1 /** 2 Copyright: © 2013 rejectedsoftware e.K. 3 License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file. 4 Authors: Sönke Ludwig 5 */ 6 module dubregistry.web; 7 8 import dubregistry.dbcontroller; 9 import dubregistry.repositories.bitbucket; 10 import dubregistry.repositories.github; 11 import dubregistry.registry; 12 import dubregistry.viewutils; // dummy import to make rdmd happy 13 14 import dub.semver; 15 import std.algorithm : sort, startsWith; 16 import std.array; 17 import std.file; 18 import std.path; 19 import std.string; 20 import userman.web; 21 import vibe.d; 22 23 24 DubRegistryWebFrontend registerDubRegistryWebFrontend(URLRouter router, DubRegistry registry, UserManController userman) 25 { 26 DubRegistryWebFrontend webfrontend; 27 if (userman) { 28 auto ff = new DubRegistryFullWebFrontend(registry, userman); 29 webfrontend = ff; 30 router.registerWebInterface(ff); 31 router.registerUserManWebInterface(userman); 32 } else { 33 webfrontend = new DubRegistryWebFrontend(registry, userman); 34 router.registerWebInterface(webfrontend); 35 } 36 router.get("*", serveStaticFiles("./public")); 37 return webfrontend; 38 } 39 40 class DubRegistryWebFrontend { 41 protected { 42 DubRegistry m_registry; 43 UserManController m_userman; 44 Category[] m_categories; 45 Category[string] m_categoryMap; 46 DbPackage[] m_packages; 47 } 48 49 this(DubRegistry registry, UserManController userman) 50 { 51 m_registry = registry; 52 m_userman = userman; 53 updateCategories(); 54 updatePackageList(); 55 } 56 57 @path("/") 58 void getHome(string sort = "updated", string category = null, ulong skip = 0, ulong limit = 20) 59 { 60 import std.algorithm.comparison : min; 61 import std.algorithm.iteration : filter, map; 62 63 static struct Info { 64 Json[] packages; 65 size_t packageCount; 66 size_t skip; 67 size_t limit; 68 Category[] categories; 69 Category[string] categoryMap; 70 } 71 72 static import std.algorithm.sorting; 73 import std.algorithm.searching : any; 74 75 DbPackage[] packages; 76 if (category.length) { 77 packages = m_packages 78 .filter!(p => p.categories.any!(c => c.startsWith(category))) 79 .array; 80 } else { 81 packages = m_packages.dup; 82 } 83 84 // sort by date of last version 85 SysTime getDate(in ref DbPackage p) { 86 if (p.versions.length == 0) return SysTime(0, UTC()); 87 return p.versions[$-1].date; 88 } 89 SysTime getDateAdded(in ref DbPackage p) { return (cast(BsonObjectID*)&p._id).timeStamp; } 90 bool compare(in ref DbPackage a, in ref DbPackage b) { 91 bool a_has_ver = a.versions.any!(v => !v.version_.startsWith("~")); 92 bool b_has_ver = b.versions.any!(v => !v.version_.startsWith("~")); 93 if (a_has_ver != b_has_ver) return a_has_ver; 94 return getDate(a) > getDate(b); 95 } 96 switch (sort) { 97 default: std.algorithm.sorting.sort!compare(packages); break; 98 case "name": std.algorithm.sorting.sort!((a, b) => a.name < b.name)(packages); break; 99 case "added": std.algorithm.sorting.sort!((a, b) => getDateAdded(a) > getDateAdded(b))(packages); break; 100 } 101 102 Info info; 103 info.packageCount = packages.length; 104 info.packages = packages[min(skip, $) .. min(skip + limit, $)] 105 .map!(p => m_registry.getPackageInfo(p, false).info) 106 .array; 107 info.skip = skip; 108 info.limit = limit; 109 info.categories = m_categories; 110 info.categoryMap = m_categoryMap; 111 112 render!("home.dt", info); 113 } 114 115 // compatibility route 116 void getAvailable(HTTPServerRequest req, HTTPServerResponse res) 117 { 118 res.redirect("/packages/index.json"); 119 } 120 121 @path("/packages/index.json") 122 void getPackages(HTTPServerRequest req, HTTPServerResponse res) 123 { 124 res.writeJsonBody(m_registry.availablePackages.array); 125 } 126 127 @path("/view_package/:packname") 128 void getRedirectViewPackage(string _packname) 129 { 130 redirect("/packages/"~_packname); 131 } 132 133 @path("/packages/:packname") 134 void getPackage(HTTPServerRequest req, HTTPServerResponse res, string _packname) 135 { 136 getPackageVersion(req, res, _packname, null); 137 } 138 139 @path("/packages/:packname/:version") 140 void getPackageVersion(HTTPServerRequest req, HTTPServerResponse res, string _packname, string _version) 141 { 142 import std.algorithm : canFind; 143 144 auto pname = _packname; 145 auto ver = _version.replace(" ", "+"); 146 string ext; 147 148 if (_version.length) { 149 if (ver.endsWith(".zip")) ext = "zip", ver = ver[0 .. $-4]; 150 else if( ver.endsWith(".json") ) ext = "json", ver = ver[0 .. $-5]; 151 } else { 152 if (pname.endsWith(".json")) { 153 pname = pname[0 .. $-5]; 154 ext = "json"; 155 } 156 } 157 158 PackageInfo packinfo; 159 PackageVersionInfo verinfo; 160 if (!getPackageInfo(pname, ver, packinfo, verinfo)) 161 return; 162 163 auto packageInfo = packinfo.info; 164 auto versionInfo = verinfo.info; 165 166 User user; 167 if (m_userman) { 168 try user = m_userman.getUser(User.ID.fromString(packageInfo["owner"].get!string)); 169 catch (Exception e) { 170 logDebug("Failed to get owner '%s' for %s %s: %s", 171 packageInfo["owner"].get!string, pname, ver, e.msg); 172 } 173 } 174 175 if (ext == "zip") { 176 if (pname.canFind(":")) return; 177 178 // This log line is a weird workaround to make otherwise undefined Json fields 179 // available. Smells like a compiler bug. 180 logDebug("%s %s", packageInfo["id"].toString(), verinfo.downloadURL); 181 182 // add download to statistic 183 m_registry.addDownload(BsonObjectID.fromString(packageInfo["id"].get!string), ver, req.headers.get("User-agent", null)); 184 if (verinfo.downloadURL.length > 0) { 185 // redirect to hosting service specific URL 186 redirect(verinfo.downloadURL); 187 } else { 188 // directly forward from hoster 189 res.headers["Content-Disposition"] = "attachment; filename=\""~pname~"-"~(ver.startsWith("~") ? ver[1 .. $] : ver) ~ ".zip\""; 190 m_registry.downloadPackageZip(pname, ver.startsWith("~") ? ver : "v"~ver, (scope data) { 191 res.writeBody(data, "application/zip"); 192 }); 193 } 194 } else if (ext == "json") { 195 if (pname.canFind(":")) return; 196 res.writeJsonBody(_version.length ? versionInfo : packageInfo); 197 } else { 198 auto gitVer = verinfo.version_; 199 gitVer = gitVer.startsWith("~") ? gitVer[1 .. $] : "v"~gitVer; 200 string urlFilter(string url, bool is_image) 201 { 202 if (url.startsWith("http://") || url.startsWith("https://")) 203 return url; 204 205 if (auto pr = "repository" in packageInfo) { 206 auto owner = (*pr)["owner"].get!string; 207 auto project = (*pr)["project"].get!string; 208 switch ((*pr)["kind"].get!string) { 209 default: return url; 210 // TODO: BitBucket + GitLab 211 case "github": 212 if (is_image) return format("https://github.com/%s/%s/raw/%s/%s", owner, project, gitVer, url); 213 else return format("https://github.com/%s/%s/blob/%s/%s", owner, project, gitVer, url); 214 } 215 } 216 217 return url; 218 } 219 220 auto packageName = pname; 221 auto registry = m_registry; 222 auto readmeContents = m_registry.getReadme(versionInfo, packageInfo["repository"].deserializeJson!DbRepository); 223 render!("view_package.dt", packageName, user, packageInfo, versionInfo, readmeContents, urlFilter, registry); 224 } 225 } 226 227 @path("/packages/:packname/versions") 228 void getAllPackageVersions(HTTPServerRequest req, HTTPServerResponse res, string _packname) { 229 import std.algorithm : canFind; 230 231 auto pname = _packname; 232 233 auto ppath = _packname.urlDecode().split(":"); 234 auto packageInfo = m_registry.getPackageInfo(ppath[0]); 235 236 auto packageName = pname; 237 auto registry = m_registry; 238 render!("view_package.versions.dt", 239 packageName, 240 packageInfo, 241 registry); 242 } 243 244 private bool getPackageInfo(string pack_name, string pack_version, out PackageInfo pkg_info, out PackageVersionInfo ver_info) { 245 import std.algorithm : map; 246 auto ppath = pack_name.urlDecode().split(":"); 247 248 pkg_info = m_registry.getPackageInfo(ppath[0]); 249 if (pkg_info.info.type == Json.Type.null_) return false; 250 251 if (pack_version.length) { 252 foreach (ref v; pkg_info.versions) { 253 if (v.version_ == pack_version) { 254 ver_info = v; 255 break; 256 } 257 } 258 if (ver_info.info.type != Json.Type.Object) return false; 259 } else { 260 import dubregistry.viewutils; 261 if (pkg_info.versions.length == 0) return false; 262 auto vidx = getBestVersionIndex(pkg_info.versions.map!(v => v.version_)); 263 ver_info = pkg_info.versions[vidx]; 264 } 265 266 foreach (i; 1 .. ppath.length) { 267 if ("subPackages" !in ver_info.info) return false; 268 bool found = false; 269 foreach (sp; ver_info.info["subPackages"]) { 270 if (sp["name"] == ppath[i]) { 271 Json newv = Json.emptyObject; 272 // inherit certain fields 273 foreach (field; ["version", "date", "license", "authors", "homepage"]) 274 if (auto pv = field in ver_info.info) newv[field] = *pv; 275 // copy/overwrite the rest frmo the sub package 276 foreach (string name, value; sp) newv[name] = value; 277 ver_info.info = newv; 278 found = true; 279 break; 280 } 281 } 282 if (!found) return false; 283 } 284 return true; 285 } 286 287 private void updateCategories() 288 { 289 auto catfile = openFile("categories.json"); 290 scope(exit) catfile.close(); 291 auto json = parseJsonString(catfile.readAllUTF8()); 292 293 Category[string] catmap; 294 295 Category processNode(Json node, string[] path) 296 { 297 path ~= node["name"].get!string; 298 auto cat = new Category; 299 cat.name = path.join("."); 300 cat.description = node["description"].get!string; 301 if (path.length > 2) 302 cat.indentedDescription = "\u00a0\u00a0\u00a0\u00a0".replicate(path.length-2) ~ "\u00a0└ " ~ cat.description; 303 else if (path.length == 2) 304 cat.indentedDescription = "\u00a0└ " ~ cat.description; 305 else cat.indentedDescription = cat.description; 306 foreach_reverse (i; 0 .. path.length) 307 if (existsFile("public/images/categories/"~path[0 .. i+1].join(".")~".png")) { 308 cat.imageName = path[0 .. i+1].join("."); 309 break; 310 } 311 312 catmap[cat.name] = cat; 313 314 if ("categories" in node) 315 foreach (subcat; node["categories"]) 316 cat.subCategories ~= processNode(subcat, path); 317 318 return cat; 319 } 320 321 Category[] cats; 322 foreach (top_level_cat; json) 323 cats ~= processNode(top_level_cat, null); 324 325 m_categories = cats; 326 m_categoryMap = catmap; 327 } 328 329 private void updatePackageList() 330 { 331 setTimer(30.seconds, &updatePackageList); 332 m_packages = m_registry.getPackageDump().array; 333 } 334 } 335 336 class DubRegistryFullWebFrontend : DubRegistryWebFrontend { 337 private { 338 UserManWebAuthenticator m_usermanauth; 339 } 340 341 this(DubRegistry registry, UserManController userman) 342 { 343 super(registry, userman); 344 m_usermanauth = new UserManWebAuthenticator(userman); 345 } 346 347 void querySearch(string q = "") 348 { 349 auto results = m_registry.searchPackages(q); 350 auto queryString = q; 351 render!("search_results.dt", queryString, results); 352 } 353 354 void getGettingStarted() { render!("getting_started.dt"); } 355 void getAbout() { redirect("/getting_started"); } 356 void getUsage() { redirect("/getting_started"); } 357 358 void getPublish() { render!("publish.dt"); } 359 void getDevelop() { render!("develop.dt"); } 360 361 @path("/package-format") 362 void getPackageFormat(string lang = null) 363 { 364 switch (lang) { 365 default: redirect("package-format?lang=json"); break; 366 case "json": render!("package_format_json.dt"); break; 367 case "sdl": render!("package_format_sdl.dt"); break; 368 } 369 } 370 371 private auto downloadInfo() 372 { 373 static struct DownloadFile { 374 string fileName; 375 string platformCaption; 376 string typeCaption; 377 } 378 379 static struct DownloadVersion { 380 string id; 381 DownloadFile[][string] files; 382 } 383 384 static struct Info { 385 DownloadVersion[] versions; 386 string latest = ""; 387 388 void addFile(string ver, string platform, string filename) 389 { 390 391 auto df = DownloadFile(filename); 392 switch (platform) { 393 default: 394 auto pts = platform.split("-"); 395 df.platformCaption = format("%s%s (%s)", pts[0][0 .. 1].toUpper(), pts[0][1 .. $], pts[1].replace("_", "-").toUpper()); 396 break; 397 case "osx-x86": df.platformCaption = "OS X (X86)"; break; 398 case "osx-x86_64": df.platformCaption = "OS X (X86-64)"; break; 399 } 400 401 if (filename.endsWith(".tar.gz")) df.typeCaption = "binary tarball"; 402 else if (filename.endsWith(".zip")) df.typeCaption = "zipped binaries"; 403 else if (filename.endsWith(".rpm")) df.typeCaption = "binary RPM package"; 404 else if (filename.endsWith("setup.exe")) df.typeCaption = "installer"; 405 else df.typeCaption = "Unknown"; 406 407 foreach(ref v; versions) 408 if( v.id == ver ){ 409 v.files[platform] ~= df; 410 return; 411 } 412 DownloadVersion dv = DownloadVersion(ver); 413 dv.files[platform] = [df]; 414 versions ~= dv; 415 if (!isPreReleaseVersion(ver) && (latest.empty || compareVersions(ver, latest) > 0)) 416 latest = ver; 417 } 418 } 419 420 Info info; 421 422 if (!"public/files".exists || !"public/files".isDir) 423 return info; 424 425 import std.regex; 426 Regex!char[][string] platformPatterns; 427 platformPatterns["windows-x86"] = [ 428 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.*))?(?:-setup\\.exe|-windows-x86\\.zip)$") 429 ]; 430 platformPatterns["linux-x86_64"] = [ 431 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86_64\\.tar\\.gz$"), 432 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86_64\\.rpm$") 433 ]; 434 platformPatterns["linux-x86"] = [ 435 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86\\.tar\\.gz$"), 436 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86\\.rpm$") 437 ]; 438 platformPatterns["linux-arm"] = [ 439 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-arm\\.tar\\.gz$"), 440 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.arm\\.rpm$") 441 ]; 442 platformPatterns["osx-x86_64"] = [ 443 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-osx-x86_64\\.tar\\.gz$"), 444 ]; 445 446 foreach(de; dirEntries("public/files", "*.*", SpanMode.shallow)) { 447 auto name = Path(de.name).head.toString(); 448 449 foreach (platform, rexes; platformPatterns) { 450 foreach (rex; rexes) { 451 auto match = match(name, rex).captures;//matchFirst(name, rex); 452 if (match.empty) continue; 453 auto ver = match["version"] ~ (match["prerelease"].length ? "-" ~ match["prerelease"] : ""); 454 if (!ver.isValidVersion()) continue; 455 info.addFile(ver, platform, name); 456 } 457 } 458 } 459 460 info.versions.sort!((a, b) => vcmp(a.id, b.id))(); 461 return info; 462 } 463 464 void getDownload() 465 { 466 auto info = downloadInfo(); 467 render!("download.dt", info); 468 } 469 470 @path("/download/LATEST") 471 void getLatest(HTTPServerResponse res) 472 { 473 auto info = downloadInfo(); 474 enforceHTTP(!info.latest.empty, HTTPStatus.notFound, "No version available."); 475 res.writeBody(info.latest); 476 } 477 478 479 @auth 480 void getMyPackages(User _user) 481 { 482 auto user = _user; 483 auto registry = m_registry; 484 render!("my_packages.dt", user, registry); 485 } 486 487 @auth @path("/register_package") 488 void getRegisterPackage(User _user, string kind = null, string owner = null, string project = null, string _error = null) 489 { 490 auto user = _user; 491 string error = _error; 492 auto registry = m_registry; 493 render!("my_packages.register.dt", user, kind, owner, project, error, registry); 494 } 495 496 @auth @path("/register_package") @errorDisplay!getRegisterPackage 497 void postRegisterPackage(string kind, string owner, string project, User _user, bool ignore_fork = false) 498 { 499 DbRepository rep; 500 rep.kind = kind; 501 rep.owner = owner; 502 rep.project = project; 503 504 if (!ignore_fork) { 505 auto info = m_registry.getRepositoryInfo(rep); 506 if (info.isFork) { 507 render!("my_packages.register.warn_fork.dt", kind, owner, project); 508 return; 509 } 510 } 511 512 m_registry.addPackage(rep, _user.id); 513 redirect("/my_packages"); 514 } 515 516 @auth @path("/my_packages/:packname") 517 void getMyPackagesPackage(string _packname, User _user, string _error = null) 518 { 519 enforceUserPackage(_user, _packname); 520 auto packageName = _packname; 521 auto nfo = m_registry.getPackageInfo(packageName); 522 if (nfo.info.type == Json.Type.null_) return; 523 auto categories = m_categories; 524 auto registry = m_registry; 525 auto user = _user; 526 auto error = _error; 527 render!("my_packages.package.dt", packageName, categories, user, registry, error); 528 } 529 530 @auth @path("/my_packages/:packname/update") 531 void postUpdatePackage(string _packname, User _user) 532 { 533 enforceUserPackage(_user, _packname); 534 m_registry.triggerPackageUpdate(_packname); 535 redirect("/my_packages/"~_packname); 536 } 537 538 @auth @path("/my_packages/:packname/remove") 539 void postShowRemovePackage(string _packname, User _user) 540 { 541 auto packageName = _packname; 542 auto user = _user; 543 enforceUserPackage(user, packageName); 544 render!("my_packages.remove.dt", packageName, user); 545 } 546 547 @auth @path("/my_packages/:packname/remove_confirm") 548 void postRemovePackage(string _packname, User _user) 549 { 550 enforceUserPackage(_user, _packname); 551 m_registry.removePackage(_packname, _user.id); 552 redirect("/my_packages"); 553 } 554 555 @auth @path("/my_packages/:packname/set_categories") 556 void postSetPackageCategories(string[] categories, string _packname, User _user) 557 { 558 enforceUserPackage(_user, _packname); 559 string[] uniquecategories; 560 outer: foreach (cat; categories) { 561 if (!cat.length) continue; 562 foreach (j, ec; uniquecategories) { 563 if (cat.startsWith(ec)) continue outer; 564 if (ec.startsWith(cat)) { 565 uniquecategories[j] = cat; 566 continue outer; 567 } 568 } 569 uniquecategories ~= cat; 570 } 571 m_registry.setPackageCategories(_packname, uniquecategories); 572 573 redirect("/my_packages/"~_packname); 574 } 575 576 @auth @path("/my_packages/:packname/set_repository") @errorDisplay!getMyPackagesPackage 577 void postSetPackageRepository(string kind, string owner, string project, string _packname, User _user) 578 { 579 enforceUserPackage(_user, _packname); 580 581 DbRepository rep; 582 rep.kind = kind; 583 rep.owner = owner; 584 rep.project = project; 585 m_registry.setPackageRepository(_packname, rep); 586 587 redirect("/my_packages/"~_packname); 588 } 589 590 @path("/docs/commandline") 591 void getCommandLineDocs() 592 { 593 import dub.commandline; 594 auto commands = getCommands(); 595 render!("docs.commandline.dt", commands); 596 } 597 598 private void enforceUserPackage(User user, string package_name) 599 { 600 enforceHTTP(m_registry.isUserPackage(user.id, package_name), HTTPStatus.forbidden, "You don't have access rights for this package."); 601 } 602 603 // Attribute for authenticated routes 604 private enum auth = before!performAuth("_user"); 605 mixin PrivateAccessProxy; 606 607 private User performAuth(HTTPServerRequest req, HTTPServerResponse res) 608 { 609 return m_usermanauth.performAuth(req, res); 610 } 611 } 612 613 final class Category { 614 string name, description, indentedDescription, imageName; 615 Category[] subCategories; 616 }