1 /** 2 Copyright: © 2013 rejectedsoftware e.K. 3 License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file. 4 Authors: Sönke Ludwig 5 */ 6 module app; 7 8 import dubregistry.dbcontroller; 9 import dubregistry.mirror; 10 import dubregistry.repositories.bitbucket; 11 import dubregistry.repositories.github; 12 import dubregistry.repositories.gitlab; 13 import dubregistry.registry; 14 import dubregistry.web; 15 import dubregistry.api; 16 17 import std.algorithm : sort; 18 import std.process : environment; 19 import std.file; 20 import std.path; 21 import std.traits : isIntegral; 22 import userman.db.controller : UserManController, createUserManController; 23 import userman.userman : UserManSettings; 24 import userman.web; 25 import vibe.d; 26 27 28 Task s_checkTask; 29 DubRegistry s_registry; 30 DubRegistryWebFrontend s_web; 31 string s_mirror; 32 33 void checkForNewVersions() 34 { 35 if (s_mirror.length) s_registry.mirrorRegistry(s_mirror); 36 else s_registry.updatePackages(); 37 } 38 39 void startMonitoring() 40 { 41 void monitorPackages() 42 { 43 sleep(1.seconds()); // give the cache a chance to warm up first 44 while(true){ 45 checkForNewVersions; 46 sleep(30.minutes()); 47 } 48 } 49 s_checkTask = runTask(&monitorPackages); 50 } 51 52 version (linux) private immutable string certPath; 53 54 shared static this() 55 { 56 enum debianCA = "/etc/ssl/certs/ca-certificates.crt"; 57 enum redhatCA = "/etc/pki/tls/certs/ca-bundle.crt"; 58 certPath = redhatCA.exists ? redhatCA : debianCA; 59 } 60 61 // generate dummy data for e.g. Heroku's preview apps 62 void defaultInit(UserManController userMan, DubRegistry registry) 63 { 64 if (environment.get("GENERATE_DEFAULT_DATA", "0") == "1" && 65 registry.getPackageDump().empty && userMan.getUserCount() == 0) 66 { 67 logInfo("'GENERATE_DEFAULT_DATA' is set and an empty database has been detected. Inserting dummy data."); 68 auto userId = userMan.registerUser("dummy@dummy.org", "dummyuser", 69 "Dummy User", "test1234"); 70 auto packages = [ 71 "https://github.com/libmir/mir-algorithm", 72 "https://github.com/libmir/mir-runtime", 73 "https://github.com/libmir/mir-random", 74 "https://github.com/libmir/mir-core", 75 "https://gitlab.com/WebFreak001/bancho-irc", 76 ]; 77 foreach (url; packages) 78 { 79 DbRepository repo; 80 repo.parseURL(URL(url)); 81 registry.addPackage(repo, userId); 82 } 83 } 84 } 85 86 struct AppConfig 87 { 88 string ghauth, 89 glurl, glauth, 90 bbuser, bbpassword; 91 92 bool enforceCertificateTrust = false; 93 94 string serviceName = "DUB - The D package registry"; 95 string serviceURL = "https://code.dlang.org/"; 96 string serviceEmail = "noreply@rejectedsoftware.com"; 97 98 string mailServer; 99 ushort mailServerPort; 100 SMTPConnectionType mailConnectionType; 101 string mailClientName; 102 string mailUser; 103 string mailPassword; 104 105 106 void init() 107 { 108 import dub.internal.utils : jsonFromFile; 109 auto regsettingsjson = jsonFromFile(NativePath("settings.json"), true); 110 // TODO: use UDAs instead 111 static immutable variables = [ 112 ["ghauth", "github-auth"], 113 ["glurl", "gitlab-url"], 114 ["glauth", "gitlab-auth"], 115 ["bbuser", "bitbucket-user"], 116 ["bbpassword", "bitbucket-password"], 117 ["enforceCertificateTrust", "enforce-certificate-trust"], 118 ["serviceName", "service-name"], 119 ["serviceURL", "service-url"], 120 ["serviceEmail", "service-email"], 121 ["mailServer", "mail-server"], 122 ["mailServerPort", "mail-server-port"], 123 ["mailConnectionType", "mail-connection-type"], 124 ["mailClientName", "mail-client-name"], 125 ["mailUser", "mail-user"], 126 ["mailPassword", "mail-password"], 127 ]; 128 static foreach (var; variables) {{ 129 alias T = typeof(__traits(getMember, this, var[0])); 130 131 T val = __traits(getMember, this, var[0]); 132 133 if (var[1] in regsettingsjson) { 134 static if (is(T == bool)) val = regsettingsjson[var[1]].get!bool; 135 else static if (isIntegral!T && !is(T == enum)) val = regsettingsjson[var[1]].get!int.to!T; 136 else val = regsettingsjson[var[1]].get!string.to!T; 137 } else { 138 // fallback to environment variables 139 auto ev = environment.get(var[1].replace("-", "_").toUpper); 140 if (ev.length) val = ev.to!T; 141 } 142 143 __traits(getMember, this, var[0]) = val; 144 }} 145 } 146 } 147 148 void main() 149 { 150 bool noMonitoring, noServe; 151 152 import std.random : rndGen, unpredictableSeed; 153 rndGen.seed(unpredictableSeed); 154 155 setLogFile("log.txt", LogLevel.diagnostic); 156 157 version (linux) version (DMD) 158 { 159 // register memory error handler on heroku 160 if ("DYNO" in environment) 161 { 162 import etc.linux.memoryerror : registerMemoryErrorHandler; 163 registerMemoryErrorHandler(); 164 } 165 } 166 167 string hostname = "code.dlang.org"; 168 169 readOption("mirror", &s_mirror, "URL of a package registry that this instance should mirror (WARNING: will overwrite local database!)"); 170 readOption("hostname", &hostname, "Domain name of this instance (default: code.dlang.org)"); 171 readOption("no-monitoring", &noMonitoring, "Don't periodically monitor for updates (for local development)"); 172 readOption("no-serve", &noServe, "Just poll for updates and exit"); 173 174 // validate provided mirror URL 175 if (s_mirror.length) 176 validateMirrorURL(s_mirror); 177 178 AppConfig appConfig; 179 appConfig.init(); 180 181 version (linux) { 182 if (appConfig.enforceCertificateTrust) { 183 logInfo("Enforcing certificate trust."); 184 HTTPClient.setTLSSetupCallback((ctx) { 185 ctx.useTrustedCertificateFile(certPath); 186 ctx.peerValidationMode = TLSPeerValidationMode.trustedCert; 187 }); 188 } 189 } 190 191 GithubRepository.register(appConfig.ghauth); 192 BitbucketRepository.register(appConfig.bbuser, appConfig.bbpassword); 193 if (appConfig.glurl.length) GitLabRepository.register(appConfig.glauth, appConfig.glurl); 194 195 auto router = new URLRouter; 196 if (s_mirror.length) router.any("*", (req, res) { req.params["mirror"] = s_mirror; }); 197 if (!noMonitoring) 198 router.get("*", (req, res) @trusted { if (!s_checkTask.running) startMonitoring(); }); 199 200 // init mongo 201 import dubregistry.mongodb : databaseName, mongoSettings; 202 mongoSettings(); 203 204 // VPM registry 205 auto regsettings = new DubRegistrySettings; 206 regsettings.databaseName = databaseName; 207 s_registry = new DubRegistry(regsettings); 208 209 UserManController userdb; 210 211 if (!s_mirror.length) { 212 // user management 213 auto udbsettings = new UserManSettings; 214 udbsettings.serviceName = appConfig.serviceName; 215 udbsettings.serviceURL = URL(appConfig.serviceURL); 216 udbsettings.serviceEmail = appConfig.serviceEmail; 217 udbsettings.databaseURL = environment.get("MONGODB_URI", environment.get("MONGO_URI", "mongodb://127.0.0.1:27017/vpmreg")); 218 udbsettings.requireActivation = false; 219 220 udbsettings.mailSettings.host = appConfig.mailServer; 221 udbsettings.mailSettings.port = appConfig.mailServerPort; 222 udbsettings.mailSettings.connectionType = appConfig.mailConnectionType; 223 udbsettings.mailSettings.localname = appConfig.mailClientName; 224 udbsettings.mailSettings.username = appConfig.mailUser; 225 udbsettings.mailSettings.password = appConfig.mailPassword; 226 if (appConfig.mailUser.length || appConfig.mailPassword.length) 227 udbsettings.mailSettings.authType = SMTPAuthType.plain; 228 udbsettings.mailSettings.tlsValidationMode = TLSPeerValidationMode.validCert; 229 version (linux) { 230 if (appConfig.enforceCertificateTrust) { 231 udbsettings.mailSettings.tlsValidationMode = TLSPeerValidationMode.trustedCert; 232 udbsettings.mailSettings.tlsContextSetup = (ctx) { 233 ctx.useTrustedCertificateFile(certPath); 234 }; 235 } 236 } 237 238 userdb = createUserManController(udbsettings); 239 } 240 241 if (noServe) { 242 if (!noMonitoring) 243 checkForNewVersions(); 244 return; 245 } 246 247 // web front end 248 s_web = router.registerDubRegistryWebFrontend(s_registry, userdb); 249 router.registerDubRegistryAPI(s_registry); 250 251 // check whether dummy data should be loaded 252 defaultInit(userdb, s_registry); 253 254 // start the web server 255 auto settings = new HTTPServerSettings; 256 settings.hostName = hostname; 257 settings.bindAddresses = ["127.0.0.1"]; 258 settings.port = 8005; 259 settings.sessionStore = new MemorySessionStore; 260 settings.useCompressionIfPossible = true; 261 readOption("bind", &settings.bindAddresses[0], "Sets the address used for serving."); 262 readOption("port|p", &settings.port, "Sets the port used for serving."); 263 264 listenHTTP(settings, router); 265 266 // poll github for new project versions 267 if (!noMonitoring) 268 startMonitoring(); 269 runApplication(); 270 }