1 /** 2 Copyright: © 2013 rejectedsoftware e.K. 3 License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file. 4 Authors: Sönke Ludwig 5 */ 6 module dubregistry.web; 7 8 import dubregistry.dbcontroller; 9 import dubregistry.repositories.bitbucket; 10 import dubregistry.repositories.github; 11 import dubregistry.registry; 12 import dubregistry.viewutils; // dummy import to make rdmd happy 13 14 import dub.semver; 15 import std.algorithm : sort, startsWith; 16 import std.array; 17 import std.file; 18 import std.path; 19 import std.string; 20 import userman.web; 21 import vibe.d; 22 23 24 DubRegistryWebFrontend registerDubRegistryWebFrontend(URLRouter router, DubRegistry registry, UserManController userman) 25 { 26 auto webfrontend = new DubRegistryWebFrontend(registry, userman); 27 router.registerUserManWebInterface(userman); 28 router.registerWebInterface(webfrontend); 29 router.get("*", serveStaticFiles("./public")); 30 return webfrontend; 31 } 32 33 class DubRegistryWebFrontend { 34 private { 35 DubRegistry m_registry; 36 UserManController m_userman; 37 UserManWebAuthenticator m_usermanauth; 38 Category[] m_categories; 39 Category[string] m_categoryMap; 40 } 41 42 this(DubRegistry registry, UserManController userman) 43 { 44 m_registry = registry; 45 m_userman = userman; 46 m_usermanauth = new UserManWebAuthenticator(userman); 47 48 updateCategories(); 49 } 50 51 // compatibility route 52 void getAvailable(HTTPServerRequest req, HTTPServerResponse res) 53 { 54 res.writeJsonBody(m_registry.availablePackages.array); 55 } 56 57 @path("/packages/index.json") 58 void getPackages(HTTPServerRequest req, HTTPServerResponse res) 59 { 60 res.writeJsonBody(m_registry.availablePackages.array); 61 } 62 63 @path("/") 64 void getHome(string sort = "updated", string category = null) 65 { 66 // collect the package list 67 auto packapp = appender!(Json[])(); 68 packapp.reserve(200); 69 if (category.length) { 70 foreach (pname; m_registry.availablePackages) { 71 auto pack = m_registry.getPackageInfo(pname); 72 foreach (c; pack.categories) { 73 if (c.get!string.startsWith(category)) { 74 packapp.put(pack); 75 break; 76 } 77 } 78 } 79 } else { 80 foreach (pack; m_registry.availablePackages) 81 packapp.put(m_registry.getPackageInfo(pack)); 82 } 83 auto packages = packapp.data; 84 85 // sort by date of last version 86 string getDate(Json p) { 87 if( p.type != Json.Type.Object || "versions" !in p ) return null; 88 if( p.versions.length == 0 ) return null; 89 return p.versions[p.versions.length-1].date.get!string; 90 } 91 SysTime getDateAdded(Json p) { 92 return SysTime.fromISOExtString(p.dateAdded.get!string); 93 } 94 bool compare(Json a, Json b) { 95 bool a_has_ver = a.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~")); 96 bool b_has_ver = b.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~")); 97 if (a_has_ver != b_has_ver) return a_has_ver; 98 return getDate(a) > getDate(b); 99 } 100 switch (sort) { 101 default: std.algorithm.sort!((a, b) => compare(a, b))(packages); break; 102 case "name": std.algorithm.sort!((a, b) => a.name < b.name)(packages); break; 103 case "added": std.algorithm.sort!((a, b) => getDateAdded(a) > getDateAdded(b))(packages); break; 104 } 105 106 auto categories = m_categories; 107 auto categoryMap = m_categoryMap; 108 render!("home.dt", categories, categoryMap, packages); 109 } 110 111 112 void querySearch(string q = "") 113 { 114 auto results = m_registry.searchPackages(q); 115 auto queryString = q; 116 render!("search_results.dt", queryString, results); 117 } 118 119 void getGettingStarted() { render!("getting_started.dt"); } 120 void getAbout() { redirect("/getting_started"); } 121 void getUsage() { redirect("/getting_started"); } 122 123 void getPublish() { render!("publish.dt"); } 124 void getDevelop() { render!("develop.dt"); } 125 126 @path("/package-format") 127 void getPackageFormat(string lang = null) 128 { 129 switch (lang) { 130 default: redirect("package-format?lang=sdl"); break; 131 case "json": render!("package_format_json.dt"); break; 132 case "sdl": render!("package_format_sdl.dt"); break; 133 } 134 } 135 136 private auto downloadInfo() 137 { 138 static struct DownloadFile { 139 string fileName; 140 string platformCaption; 141 string typeCaption; 142 } 143 144 static struct DownloadVersion { 145 string id; 146 DownloadFile[][string] files; 147 } 148 149 static struct Info { 150 DownloadVersion[] versions; 151 string latest = ""; 152 153 void addFile(string ver, string platform, string filename) 154 { 155 156 auto df = DownloadFile(filename); 157 switch (platform) { 158 default: 159 auto pts = platform.split("-"); 160 df.platformCaption = format("%s%s (%s)", pts[0][0 .. 1].toUpper(), pts[0][1 .. $], pts[1].replace("_", "-").toUpper()); 161 break; 162 case "osx-x86": df.platformCaption = "OS X (X86)"; break; 163 case "osx-x86_64": df.platformCaption = "OS X (X86-64)"; break; 164 } 165 166 if (filename.endsWith(".tar.gz")) df.typeCaption = "binary tarball"; 167 else if (filename.endsWith(".zip")) df.typeCaption = "zipped binaries"; 168 else if (filename.endsWith(".rpm")) df.typeCaption = "binary RPM package"; 169 else if (filename.endsWith("setup.exe")) df.typeCaption = "installer"; 170 else df.typeCaption = "Unknown"; 171 172 foreach(ref v; versions) 173 if( v.id == ver ){ 174 v.files[platform] ~= df; 175 return; 176 } 177 DownloadVersion dv = DownloadVersion(ver); 178 dv.files[platform] = [df]; 179 versions ~= dv; 180 if (!isPreReleaseVersion(ver) && (latest.empty || compareVersions(ver, latest) > 0)) 181 latest = ver; 182 } 183 } 184 185 Info info; 186 187 if (!"public/files".exists || !"public/files".isDir) 188 return info; 189 190 import std.regex; 191 Regex!char[][string] platformPatterns; 192 platformPatterns["windows-x86"] = [ 193 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.*))?(?:-setup\\.exe|-windows-x86\\.zip)$") 194 ]; 195 platformPatterns["linux-x86_64"] = [ 196 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86_64\\.tar\\.gz$"), 197 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86_64\\.rpm$") 198 ]; 199 platformPatterns["linux-x86"] = [ 200 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86\\.tar\\.gz$"), 201 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86\\.rpm$") 202 ]; 203 platformPatterns["linux-arm"] = [ 204 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-arm\\.tar\\.gz$"), 205 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.arm\\.rpm$") 206 ]; 207 platformPatterns["osx-x86_64"] = [ 208 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-osx-x86_64\\.tar\\.gz$"), 209 ]; 210 211 foreach(de; dirEntries("public/files", "*.*", SpanMode.shallow)) { 212 auto name = Path(de.name).head.toString(); 213 214 foreach (platform, rexes; platformPatterns) { 215 foreach (rex; rexes) { 216 auto match = match(name, rex).captures;//matchFirst(name, rex); 217 if (match.empty) continue; 218 auto ver = match["version"] ~ (match["prerelease"].length ? "-" ~ match["prerelease"] : ""); 219 if (!ver.isValidVersion()) continue; 220 info.addFile(ver, platform, name); 221 } 222 } 223 } 224 225 info.versions.sort!((a, b) => vcmp(a.id, b.id))(); 226 return info; 227 } 228 229 void getDownload() 230 { 231 auto info = downloadInfo(); 232 render!("download.dt", info); 233 } 234 235 @path("/download/LATEST") 236 void getLatest(HTTPServerResponse res) 237 { 238 auto info = downloadInfo(); 239 enforceHTTP(!info.latest.empty, HTTPStatus.notFound, "No version available."); 240 res.writeBody(info.latest); 241 } 242 243 @path("/view_package/:packname") 244 void getRedirectViewPackage(string _packname) 245 { 246 redirect("/packages/"~_packname); 247 } 248 249 @path("/packages/:packname") 250 void getPackage(HTTPServerResponse res, string _packname) 251 { 252 bool json = false; 253 auto pname = _packname; 254 if( pname.endsWith(".json") ){ 255 pname = pname[0 .. $-5]; 256 json = true; 257 } 258 259 Json packageInfo, versionInfo; 260 if (!getPackageInfo(pname, null, packageInfo, versionInfo)) 261 return; 262 263 auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string)); 264 265 if (json) { 266 if (pname.canFind(":")) return; 267 res.writeJsonBody(packageInfo); 268 } else { 269 string packageName = pname; 270 render!("view_package.dt", packageName, user, packageInfo, versionInfo); 271 } 272 } 273 274 @path("/packages/:packname/:version") 275 void getPackageVersion(HTTPServerRequest req, HTTPServerResponse res, string _packname, string _version) 276 { 277 auto pname = _packname; 278 279 auto ver = req.params["version"].replace(" ", "+"); 280 string ext; 281 if( ver.endsWith(".zip") ) ext = "zip", ver = ver[0 .. $-4]; 282 else if( ver.endsWith(".json") ) ext = "json", ver = ver[0 .. $-5]; 283 284 Json packageInfo, versionInfo; 285 if (!getPackageInfo(pname, ver, packageInfo, versionInfo)) 286 return; 287 288 auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string)); 289 290 if (ext == "zip") { 291 if (pname.canFind(":")) return; 292 293 // This log line is a weird workaround to make otherwise undefined Json fields 294 // available. Smells like a compiler bug. 295 logDebug("%s %s", packageInfo["id"].toString(), versionInfo["downloadUrl"].toString()); 296 297 // add download to statistic 298 m_registry.addDownload(BsonObjectID.fromString(packageInfo.id.get!string), ver, req.headers.get("User-agent", null)); 299 if (versionInfo["downloadUrl"].get!string.length > 0) { 300 // redirect to hosting service specific URL 301 redirect(versionInfo.downloadUrl.get!string); 302 } else { 303 // directly forward from hoster 304 res.headers["Content-Disposition"] = "attachment; filename=\""~pname~"-"~(ver.startsWith("~") ? ver[1 .. $] : ver) ~ ".zip\""; 305 m_registry.downloadPackageZip(pname, ver.startsWith("~") ? ver : "v"~ver, (scope data) { 306 res.writeBody(data, "application/zip"); 307 }); 308 } 309 } else if ( ext == "json") { 310 if (pname.canFind(":")) return; 311 res.writeJsonBody(versionInfo); 312 } else { 313 auto packageName = pname; 314 render!("view_package.dt", packageName, user, packageInfo, versionInfo); 315 } 316 } 317 318 private bool getPackageInfo(string pack_name, string pack_version, out Json pkg_info, out Json ver_info) 319 { 320 auto ppath = pack_name.urlDecode().split(":"); 321 322 pkg_info = m_registry.getPackageInfo(ppath[0]); 323 if (pkg_info.type == Json.Type.null_) return false; 324 325 if (pack_version.length) { 326 foreach (v; pkg_info.versions) { 327 if (v["version"].get!string == pack_version) { 328 ver_info = v; 329 break; 330 } 331 } 332 if (ver_info.type != Json.Type.Object) return false; 333 } else { 334 import dubregistry.viewutils; 335 if (pkg_info.versions.length == 0) return false; 336 ver_info = getBestVersion(pkg_info.versions); 337 } 338 339 foreach (i; 1 .. ppath.length) { 340 if ("subPackages" !in ver_info) return false; 341 bool found = false; 342 foreach (sp; ver_info.subPackages) { 343 if (sp.name == ppath[i]) { 344 Json newv = Json.emptyObject; 345 // inherit certain fields 346 foreach (field; ["version", "date", "license", "authors", "homepage"]) 347 if (auto pv = field in ver_info) newv[field] = *pv; 348 // copy/overwrite the rest frmo the sub package 349 foreach (string name, value; sp) newv[name] = value; 350 ver_info = newv; 351 found = true; 352 break; 353 } 354 } 355 if (!found) return false; 356 } 357 return true; 358 } 359 360 @auth 361 void getMyPackages(User _user) 362 { 363 auto user = _user; 364 auto registry = m_registry; 365 render!("my_packages.dt", user, registry); 366 } 367 368 @auth @path("/register_package") 369 void getRegisterPackage(User _user, string _error = null) 370 { 371 auto user = _user; 372 string error = _error; 373 auto registry = m_registry; 374 render!("my_packages.register.dt", user, error, registry); 375 } 376 377 @auth @path("/register_package") @errorDisplay!getRegisterPackage 378 void postRegisterPackage(string kind, string owner, string project, User _user) 379 { 380 Json rep = Json.emptyObject; 381 rep["kind"] = kind; 382 rep["owner"] = owner; 383 rep["project"] = project; 384 m_registry.addPackage(rep, _user.id); 385 redirect("/my_packages"); 386 } 387 388 @auth @path("/my_packages/:packname") 389 void getMyPackagesPackage(string _packname, User _user, string _error = null) 390 { 391 enforceUserPackage(_user, _packname); 392 auto packageName = _packname; 393 auto nfo = m_registry.getPackageInfo(packageName); 394 if (nfo.type == Json.Type.null_) return; 395 auto categories = m_categories; 396 auto registry = m_registry; 397 auto user = _user; 398 auto error = _error; 399 render!("my_packages.package.dt", packageName, categories, user, registry, error); 400 } 401 402 @auth @path("/my_packages/:packname/update") 403 void postUpdatePackage(string _packname, User _user) 404 { 405 enforceUserPackage(_user, _packname); 406 m_registry.triggerPackageUpdate(_packname); 407 redirect("/my_packages/"~_packname); 408 } 409 410 @auth @path("/my_packages/:packname/remove") 411 void postShowRemovePackage(string _packname, User _user) 412 { 413 auto packageName = _packname; 414 auto user = _user; 415 enforceUserPackage(user, packageName); 416 render!("my_packages.remove.dt", packageName, user); 417 } 418 419 @auth @path("/my_packages/:packname/remove_confirm") 420 void postRemovePackage(string _packname, User _user) 421 { 422 enforceUserPackage(_user, _packname); 423 m_registry.removePackage(_packname, _user.id); 424 redirect("/my_packages"); 425 } 426 427 @auth @path("/my_packages/:packname/set_categories") 428 void postSetPackageCategories(string[] categories, string _packname, User _user) 429 { 430 enforceUserPackage(_user, _packname); 431 string[] uniquecategories; 432 outer: foreach (cat; categories) { 433 if (!cat.length) continue; 434 foreach (j, ec; uniquecategories) { 435 if (cat.startsWith(ec)) continue outer; 436 if (ec.startsWith(cat)) { 437 uniquecategories[j] = cat; 438 continue outer; 439 } 440 } 441 uniquecategories ~= cat; 442 } 443 m_registry.setPackageCategories(_packname, uniquecategories); 444 445 redirect("/my_packages/"~_packname); 446 } 447 448 @auth @path("/my_packages/:packname/set_repository") @errorDisplay!getMyPackagesPackage 449 void postSetPackageRepository(string kind, string owner, string project, string _packname, User _user) 450 { 451 enforceUserPackage(_user, _packname); 452 453 Json rep = Json.emptyObject; 454 rep["kind"] = kind; 455 rep["owner"] = owner; 456 rep["project"] = project; 457 m_registry.setPackageRepository(_packname, rep); 458 459 redirect("/my_packages/"~_packname); 460 } 461 462 @path("/docs/commandline") 463 void getCommandLineDocs() 464 { 465 import dub.commandline; 466 auto commands = getCommands(); 467 render!("docs.commandline.dt", commands); 468 } 469 470 private void enforceUserPackage(User user, string package_name) 471 { 472 enforceHTTP(m_registry.isUserPackage(user.id, package_name), HTTPStatus.forbidden, "You don't have access rights for this package."); 473 } 474 475 private void updateCategories() 476 { 477 auto catfile = openFile("categories.json"); 478 scope(exit) catfile.close(); 479 auto json = parseJsonString(catfile.readAllUTF8()); 480 481 Category[string] catmap; 482 483 Category processNode(Json node, string[] path) 484 { 485 path ~= node.name.get!string; 486 auto cat = new Category; 487 cat.name = path.join("."); 488 cat.description = node.description.get!string; 489 if (path.length > 2) 490 cat.indentedDescription = "\u00a0\u00a0\u00a0\u00a0".replicate(path.length-2) ~ "\u00a0└ " ~ cat.description; 491 else if (path.length == 2) 492 cat.indentedDescription = "\u00a0└ " ~ cat.description; 493 else cat.indentedDescription = cat.description; 494 foreach_reverse (i; 0 .. path.length) 495 if (existsFile("public/images/categories/"~path[0 .. i].join(".")~".png")) { 496 cat.imageName = path[0 .. i].join("."); 497 break; 498 } 499 500 catmap[cat.name] = cat; 501 502 if ("categories" in node) 503 foreach (subcat; node.categories) 504 cat.subCategories ~= processNode(subcat, path); 505 506 return cat; 507 } 508 509 Category[] cats; 510 foreach (top_level_cat; json) 511 cats ~= processNode(top_level_cat, null); 512 513 m_categories = cats; 514 m_categoryMap = catmap; 515 } 516 517 // Attribute for authenticated routes 518 private enum auth = before!performAuth("_user"); 519 mixin PrivateAccessProxy; 520 521 private User performAuth(HTTPServerRequest req, HTTPServerResponse res) 522 { 523 return m_usermanauth.performAuth(req, res); 524 } 525 } 526 527 final class Category { 528 string name, description, indentedDescription, imageName; 529 Category[] subCategories; 530 }