1 /** 2 */ 3 module dubregistry.mirror; 4 5 import dubregistry.registry; 6 import dubregistry.dbcontroller; 7 import userman.db.controller; 8 import vibe.core.log; 9 import vibe.data.bson; 10 import vibe.http.client; 11 import vibe.inet.url; 12 import std.array : array; 13 import std.datetime : SysTime; 14 import std.encoding : sanitize; 15 import std.format : format; 16 17 void validateMirrorURL(ref string base_url) 18 { 19 import std.exception : enforce; 20 import std.algorithm.searching : endsWith; 21 22 // ensure the URL has a trailing slash 23 if (!base_url.endsWith('/')) base_url ~= '/'; 24 25 // check two characteristic API endpoints 26 enum urls = ["packages/index.json", "api/packages/search?q=foobar"]; 27 foreach (url; urls) { 28 try { 29 requestHTTP(base_url ~ url, 30 (scope req) { req.method = HTTPMethod.HEAD; }, 31 (scope res) { 32 enforce(res.statusCode < 400, 33 format("Endpoint '%s' could not be accessed: %s", url, httpStatusText(res.statusCode))); 34 } 35 ); 36 } catch (Exception e) { 37 throw new Exception("The provided mirror URL does not appear to point to a valid DUB registry root: "~e.msg); 38 } 39 } 40 } 41 42 void mirrorRegistry(DubRegistry registry, URL url) 43 nothrow { 44 logInfo("Polling '%s' for updates...", url); 45 try { 46 auto packs = requestHTTP(url ~ Path("api/packages/dump")).readJson().deserializeJson!(DbPackage[]); 47 48 bool[BsonObjectID] current_packs; 49 foreach (p; packs) current_packs[p._id] = true; 50 51 // first, remove all packages that don't exist anymore to avoid possible name conflicts 52 foreach (id; registry.availablePackageIDs) 53 if (id !in current_packs) { 54 try { 55 auto pack = registry.db.getPackage(id); 56 logInfo("Removing package '%s", pack.name); 57 registry.removePackage(pack.name, User.ID(pack.owner)); 58 } catch (Exception e) { 59 logError("Failed to remove package with ID '%s': %s", id, e.msg); 60 logDiagnostic("Full error: %s", e.toString().sanitize); 61 } 62 } 63 64 // then add/update all existing packages 65 foreach (p; packs) { 66 try { 67 logInfo("Updating package '%s'", p.name); 68 registry.addOrSetPackage(p); 69 } catch (Exception e) { 70 logError("Failed to add/update package '%s': %s", p.name, e.msg); 71 logDiagnostic("Full error: %s", e.toString().sanitize); 72 } 73 } 74 } catch (Exception e) { 75 logError("Fetching updated packages failed: %s", e.msg); 76 logDiagnostic("Full error: %s", e.toString().sanitize); 77 } 78 }