1 /**
2 */
3 module dubregistry.mirror;
4 
5 import dubregistry.registry;
6 import dubregistry.dbcontroller;
7 import userman.db.controller;
8 import vibe.core.log;
9 import vibe.data.bson;
10 import vibe.http.client;
11 import vibe.inet.url;
12 import std.array : array;
13 import std.datetime : SysTime;
14 import std.encoding : sanitize;
15 import std.format : format;
16 
17 void validateMirrorURL(ref string base_url)
18 {
19 	import std.exception : enforce;
20 	import std.algorithm.searching : endsWith;
21 
22 	// ensure the URL has a trailing slash
23 	if (!base_url.endsWith('/')) base_url ~= '/';
24 
25 	// check two characteristic API endpoints
26 	enum urls = ["packages/index.json", "api/packages/search?q=foobar"];
27 	foreach (url; urls) {
28 		try {
29 			requestHTTP(base_url ~ url,
30 				(scope req) { req.method = HTTPMethod.HEAD; },
31 				(scope res) {
32 					enforce(res.statusCode < 400,
33 						format("Endpoint '%s' could not be accessed: %s", url, httpStatusText(res.statusCode)));
34 				}
35 			);
36 		} catch (Exception e) {
37 			throw new Exception("The provided mirror URL does not appear to point to a valid DUB registry root: "~e.msg);
38 		}
39 	}
40 }
41 
42 void mirrorRegistry(DubRegistry registry, URL url)
43 nothrow {
44 	logInfo("Polling '%s' for updates...", url);
45 	try {
46 		auto packs = requestHTTP(url ~ Path("api/packages/dump")).readJson().deserializeJson!(DbPackage[]);
47 
48 		bool[BsonObjectID] current_packs;
49 		foreach (p; packs) current_packs[p._id] = true;
50 
51 		// first, remove all packages that don't exist anymore to avoid possible name conflicts
52 		foreach (id; registry.availablePackageIDs)
53 			if (id !in current_packs) {
54 				try {
55 					auto pack = registry.db.getPackage(id);
56 					logInfo("Removing package '%s", pack.name);
57 					registry.removePackage(pack.name, User.ID(pack.owner));
58 				} catch (Exception e) {
59 					logError("Failed to remove package with ID '%s': %s", id, e.msg);
60 					logDiagnostic("Full error: %s", e.toString().sanitize);
61 				}
62 			}
63 
64 		// then add/update all existing packages
65 		foreach (p; packs) {
66 			try {
67 				logInfo("Updating package '%s'", p.name);
68 				registry.addOrSetPackage(p);
69 			} catch (Exception e) {
70 				logError("Failed to add/update package '%s': %s", p.name, e.msg);
71 				logDiagnostic("Full error: %s", e.toString().sanitize);
72 			}
73 		}
74 	} catch (Exception e) {
75 		logError("Fetching updated packages failed: %s", e.msg);
76 		logDiagnostic("Full error: %s", e.toString().sanitize);
77 	}
78 }