1 /** 2 Copyright: © 2013 rejectedsoftware e.K. 3 License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file. 4 Authors: Sönke Ludwig 5 */ 6 module dubregistry.web; 7 8 import dubregistry.dbcontroller; 9 import dubregistry.repositories.bitbucket; 10 import dubregistry.repositories.github; 11 import dubregistry.registry; 12 import dubregistry.viewutils; // dummy import to make rdmd happy 13 14 import dub.semver; 15 import std.algorithm : sort, startsWith; 16 import std.array; 17 import std.file; 18 import std.path; 19 import std.string; 20 import userman.web; 21 import vibe.d; 22 23 24 DubRegistryWebFrontend registerDubRegistryWebFrontend(URLRouter router, DubRegistry registry, UserManController userman) 25 { 26 auto webfrontend = new DubRegistryWebFrontend(registry, userman); 27 router.registerUserManWebInterface(userman); 28 router.registerWebInterface(webfrontend); 29 router.get("*", serveStaticFiles("./public")); 30 return webfrontend; 31 } 32 33 class DubRegistryWebFrontend { 34 private { 35 DubRegistry m_registry; 36 UserManController m_userman; 37 UserManWebAuthenticator m_usermanauth; 38 Category[] m_categories; 39 Category[string] m_categoryMap; 40 } 41 42 this(DubRegistry registry, UserManController userman) 43 { 44 m_registry = registry; 45 m_userman = userman; 46 m_usermanauth = new UserManWebAuthenticator(userman); 47 48 updateCategories(); 49 } 50 51 // compatibility route 52 void getAvailable(HTTPServerRequest req, HTTPServerResponse res) 53 { 54 res.writeJsonBody(m_registry.availablePackages.array); 55 } 56 57 @path("/packages/index.json") 58 void getPackages(HTTPServerRequest req, HTTPServerResponse res) 59 { 60 res.writeJsonBody(m_registry.availablePackages.array); 61 } 62 63 @path("/") 64 void getHome(string sort = "updated", string category = null) 65 { 66 // collect the package list 67 auto packapp = appender!(Json[])(); 68 packapp.reserve(200); 69 if (category.length) { 70 foreach (pname; m_registry.availablePackages) { 71 auto pack = m_registry.getPackageInfo(pname); 72 foreach (c; pack.categories) { 73 if (c.get!string.startsWith(category)) { 74 packapp.put(pack); 75 break; 76 } 77 } 78 } 79 } else { 80 foreach (pack; m_registry.availablePackages) 81 packapp.put(m_registry.getPackageInfo(pack)); 82 } 83 auto packages = packapp.data; 84 85 // sort by date of last version 86 string getDate(Json p) { 87 if( p.type != Json.Type.Object || "versions" !in p ) return null; 88 if( p.versions.length == 0 ) return null; 89 return p.versions[p.versions.length-1].date.get!string; 90 } 91 SysTime getDateAdded(Json p) { 92 return SysTime.fromISOExtString(p.dateAdded.get!string); 93 } 94 bool compare(Json a, Json b) { 95 bool a_has_ver = a.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~")); 96 bool b_has_ver = b.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~")); 97 if (a_has_ver != b_has_ver) return a_has_ver; 98 return getDate(a) > getDate(b); 99 } 100 switch (sort) { 101 default: std.algorithm.sort!((a, b) => compare(a, b))(packages); break; 102 case "name": std.algorithm.sort!((a, b) => a.name < b.name)(packages); break; 103 case "added": std.algorithm.sort!((a, b) => getDateAdded(a) > getDateAdded(b))(packages); break; 104 } 105 106 auto categories = m_categories; 107 auto categoryMap = m_categoryMap; 108 render!("home.dt", categories, categoryMap, packages); 109 } 110 111 112 void querySearch(string q = "") 113 { 114 auto queryString = q; 115 auto keywords = queryString.split(); 116 auto results = m_registry.searchPackages(keywords); 117 render!("search_results.dt", queryString, results); 118 } 119 120 void getGettingStarted() { render!("getting_started.dt"); } 121 void getAbout() { redirect("/getting_started"); } 122 void getUsage() { redirect("/getting_started"); } 123 124 void getPublish() { render!("publish.dt"); } 125 void getDevelop() { render!("develop.dt"); } 126 @path("/package-format") 127 void getPackageFormat() { render!("package_format.dt"); } 128 129 private auto downloadInfo() 130 { 131 static struct DownloadFile { 132 string fileName; 133 string platformCaption; 134 string typeCaption; 135 } 136 137 static struct DownloadVersion { 138 string id; 139 DownloadFile[][string] files; 140 } 141 142 static struct Info { 143 DownloadVersion[] versions; 144 string latest = ""; 145 146 void addFile(string ver, string platform, string filename) 147 { 148 149 auto df = DownloadFile(filename); 150 switch (platform) { 151 default: 152 auto pts = platform.split("-"); 153 df.platformCaption = format("%s%s (%s)", pts[0][0 .. 1].toUpper(), pts[0][1 .. $], pts[1].replace("_", "-").toUpper()); 154 break; 155 case "osx-x86": df.platformCaption = "OS X (X86)"; break; 156 case "osx-x86_64": df.platformCaption = "OS X (X86-64)"; break; 157 } 158 159 if (filename.endsWith(".tar.gz")) df.typeCaption = "binary tarball"; 160 else if (filename.endsWith(".zip")) df.typeCaption = "zipped binaries"; 161 else if (filename.endsWith(".rpm")) df.typeCaption = "binary RPM package"; 162 else if (filename.endsWith("setup.exe")) df.typeCaption = "installer"; 163 else df.typeCaption = "Unknown"; 164 165 foreach(ref v; versions) 166 if( v.id == ver ){ 167 v.files[platform] ~= df; 168 return; 169 } 170 DownloadVersion dv = DownloadVersion(ver); 171 dv.files[platform] = [df]; 172 versions ~= dv; 173 if (!isPreReleaseVersion(ver) && (latest.empty || compareVersions(ver, latest) > 0)) 174 latest = ver; 175 } 176 } 177 178 Info info; 179 180 if (!"public/files".exists || !"public/files".isDir) 181 return info; 182 183 import std.regex; 184 Regex!char[][string] platformPatterns; 185 platformPatterns["windows-x86"] = [ 186 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.*))?(?:-setup\\.exe|-windows-x86\\.zip)$") 187 ]; 188 platformPatterns["linux-x86_64"] = [ 189 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86_64\\.tar\\.gz$"), 190 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86_64\\.rpm$") 191 ]; 192 platformPatterns["linux-x86"] = [ 193 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86\\.tar\\.gz$"), 194 regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86\\.rpm$") 195 ]; 196 platformPatterns["osx-x86_64"] = [ 197 regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-osx-x86_64\\.tar\\.gz$"), 198 ]; 199 200 foreach(de; dirEntries("public/files", "*.*", SpanMode.shallow)) { 201 auto name = Path(de.name).head.toString(); 202 203 foreach (platform, rexes; platformPatterns) { 204 foreach (rex; rexes) { 205 auto match = match(name, rex).captures;//matchFirst(name, rex); 206 if (match.empty) continue; 207 auto ver = match["version"] ~ (match["prerelease"].length ? "-" ~ match["prerelease"] : ""); 208 if (!ver.isValidVersion()) continue; 209 info.addFile(ver, platform, name); 210 } 211 } 212 } 213 214 info.versions.sort!((a, b) => vcmp(a.id, b.id))(); 215 return info; 216 } 217 218 void getDownload() 219 { 220 auto info = downloadInfo(); 221 render!("download.dt", info); 222 } 223 224 @path("/download/LATEST") 225 void getLatest(HTTPServerResponse res) 226 { 227 auto info = downloadInfo(); 228 enforceHTTP(!info.latest.empty, HTTPStatus.notFound, "No version available."); 229 res.writeBody(info.latest); 230 } 231 232 @path("/view_package/:packname") 233 void getRedirectViewPackage(string _packname) 234 { 235 redirect("/packages/"~_packname); 236 } 237 238 @path("/packages/:packname") 239 void getPackage(HTTPServerResponse res, string _packname) 240 { 241 bool json = false; 242 auto pname = _packname; 243 if( pname.endsWith(".json") ){ 244 pname = pname[0 .. $-5]; 245 json = true; 246 } 247 248 Json packageInfo, versionInfo; 249 if (!getPackageInfo(pname, null, packageInfo, versionInfo)) 250 return; 251 252 auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string)); 253 254 if (json) { 255 if (pname.canFind(":")) return; 256 res.writeJsonBody(packageInfo); 257 } else { 258 string packageName = pname; 259 render!("view_package.dt", packageName, user, packageInfo, versionInfo); 260 } 261 } 262 263 @path("/packages/:packname/:version") 264 void getPackageVersion(HTTPServerRequest req, HTTPServerResponse res, string _packname, string _version) 265 { 266 auto pname = _packname; 267 268 auto ver = req.params["version"].replace(" ", "+"); 269 string ext; 270 if( ver.endsWith(".zip") ) ext = "zip", ver = ver[0 .. $-4]; 271 else if( ver.endsWith(".json") ) ext = "json", ver = ver[0 .. $-5]; 272 273 Json packageInfo, versionInfo; 274 if (!getPackageInfo(pname, ver, packageInfo, versionInfo)) 275 return; 276 277 auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string)); 278 279 if (ext == "zip") { 280 if (pname.canFind(":")) return; 281 // add download to statistic 282 m_registry.addDownload(BsonObjectID.fromString(packageInfo.id.get!string), ver, req.headers.get("User-agent", null)); 283 // redirect to hosting service specific URL 284 redirect(versionInfo.downloadUrl.get!string); 285 } else if ( ext == "json") { 286 if (pname.canFind(":")) return; 287 res.writeJsonBody(versionInfo); 288 } else { 289 auto packageName = pname; 290 render!("view_package.dt", packageName, user, packageInfo, versionInfo); 291 } 292 } 293 294 private bool getPackageInfo(string pack_name, string pack_version, out Json pkg_info, out Json ver_info) 295 { 296 auto ppath = pack_name.urlDecode().split(":"); 297 298 pkg_info = m_registry.getPackageInfo(ppath[0]); 299 if (pkg_info == null) return false; 300 301 if (pack_version.length) { 302 foreach (v; pkg_info.versions) { 303 if (v["version"].get!string == pack_version) { 304 ver_info = v; 305 break; 306 } 307 } 308 if (ver_info.type != Json.Type.Object) return false; 309 } else { 310 import dubregistry.viewutils; 311 if (pkg_info.versions.length == 0) return false; 312 ver_info = getBestVersion(pkg_info.versions); 313 } 314 315 foreach (i; 1 .. ppath.length) { 316 if ("subPackages" !in ver_info) return false; 317 bool found = false; 318 foreach (sp; ver_info.subPackages) { 319 if (sp.name == ppath[i]) { 320 Json newv = Json.emptyObject; 321 // inherit certain fields 322 foreach (field; ["version", "date", "license", "authors", "homepage"]) 323 if (auto pv = field in ver_info) newv[field] = *pv; 324 // copy/overwrite the rest frmo the sub package 325 foreach (string name, value; sp) newv[name] = value; 326 ver_info = newv; 327 found = true; 328 break; 329 } 330 } 331 if (!found) return false; 332 } 333 return true; 334 } 335 336 @auth 337 void getMyPackages(User _user) 338 { 339 auto user = _user; 340 auto registry = m_registry; 341 render!("my_packages.dt", user, registry); 342 } 343 344 @auth @path("/register_package") 345 void getRegisterPackage(User _user, string _error = null) 346 { 347 auto user = _user; 348 string error = _error; 349 auto registry = m_registry; 350 render!("my_packages.register.dt", user, error, registry); 351 } 352 353 @auth @path("/register_package") @errorDisplay!getRegisterPackage 354 void postRegisterPackage(string kind, string owner, string project, User _user) 355 { 356 Json rep = Json.emptyObject; 357 rep["kind"] = kind; 358 rep["owner"] = owner; 359 rep["project"] = project; 360 m_registry.addPackage(rep, _user.id); 361 redirect("/my_packages"); 362 } 363 364 @auth @path("/my_packages/:packname") 365 void getMyPackagesPackage(string _packname, User _user, string _error = null) 366 { 367 enforceUserPackage(_user, _packname); 368 auto packageName = _packname; 369 auto nfo = m_registry.getPackageInfo(packageName); 370 if (nfo.type == Json.Type.null_) return; 371 auto categories = m_categories; 372 auto registry = m_registry; 373 auto user = _user; 374 auto error = _error; 375 render!("my_packages.package.dt", packageName, categories, user, registry, error); 376 } 377 378 @auth @path("/my_packages/:packname/update") 379 void postUpdatePackage(string _packname, User _user) 380 { 381 enforceUserPackage(_user, _packname); 382 m_registry.triggerPackageUpdate(_packname); 383 redirect("/my_packages/"~_packname); 384 } 385 386 @auth @path("/my_packages/:packname/remove") 387 void postShowRemovePackage(string _packname, User _user) 388 { 389 auto packageName = _packname; 390 auto user = _user; 391 enforceUserPackage(user, packageName); 392 render!("my_packages.remove.dt", packageName, user); 393 } 394 395 @auth @path("/my_packages/:packname/remove_confirm") 396 void postRemovePackage(string _packname, User _user) 397 { 398 enforceUserPackage(_user, _packname); 399 m_registry.removePackage(_packname, _user.id); 400 redirect("/my_packages"); 401 } 402 403 @auth @path("/my_packages/:packname/set_categories") 404 void postSetPackageCategories(string[] categories, string _packname, User _user) 405 { 406 enforceUserPackage(_user, _packname); 407 string[] uniquecategories; 408 outer: foreach (cat; categories) { 409 if (!cat.length) continue; 410 foreach (j, ec; uniquecategories) { 411 if (cat.startsWith(ec)) continue outer; 412 if (ec.startsWith(cat)) { 413 uniquecategories[j] = cat; 414 continue outer; 415 } 416 } 417 uniquecategories ~= cat; 418 } 419 m_registry.setPackageCategories(_packname, uniquecategories); 420 421 redirect("/my_packages/"~_packname); 422 } 423 424 @auth @path("/my_packages/:packname/set_repository") @errorDisplay!getMyPackagesPackage 425 void postSetPackageRepository(string kind, string owner, string project, string _packname, User _user) 426 { 427 enforceUserPackage(_user, _packname); 428 429 Json rep = Json.emptyObject; 430 rep["kind"] = kind; 431 rep["owner"] = owner; 432 rep["project"] = project; 433 m_registry.setPackageRepository(_packname, rep); 434 435 redirect("/my_packages/"~_packname); 436 } 437 438 private void enforceUserPackage(User user, string package_name) 439 { 440 enforceHTTP(m_registry.isUserPackage(user.id, package_name), HTTPStatus.forbidden, "You don't have access rights for this package."); 441 } 442 443 private void updateCategories() 444 { 445 auto catfile = openFile("categories.json"); 446 scope(exit) catfile.close(); 447 auto json = parseJsonString(catfile.readAllUTF8()); 448 449 Category[string] catmap; 450 451 Category processNode(Json node, string[] path) 452 { 453 path ~= node.name.get!string; 454 auto cat = new Category; 455 cat.name = path.join("."); 456 cat.description = node.description.get!string; 457 if (path.length > 2) 458 cat.indentedDescription = "\u00a0\u00a0\u00a0\u00a0".replicate(path.length-2) ~ "\u00a0└ " ~ cat.description; 459 else if (path.length == 2) 460 cat.indentedDescription = "\u00a0└ " ~ cat.description; 461 else cat.indentedDescription = cat.description; 462 foreach_reverse (i; 0 .. path.length) 463 if (existsFile("public/images/categories/"~path[0 .. i].join(".")~".png")) { 464 cat.imageName = path[0 .. i].join("."); 465 break; 466 } 467 468 catmap[cat.name] = cat; 469 470 if ("categories" in node) 471 foreach (subcat; node.categories) 472 cat.subCategories ~= processNode(subcat, path); 473 474 return cat; 475 } 476 477 Category[] cats; 478 foreach (top_level_cat; json) 479 cats ~= processNode(top_level_cat, null); 480 481 m_categories = cats; 482 m_categoryMap = catmap; 483 } 484 485 // Attribute for authenticated routes 486 private enum auth = before!performAuth("_user"); 487 mixin PrivateAccessProxy; 488 489 private User performAuth(HTTPServerRequest req, HTTPServerResponse res) 490 { 491 return m_usermanauth.performAuth(req, res); 492 } 493 } 494 495 final class Category { 496 string name, description, indentedDescription, imageName; 497 Category[] subCategories; 498 }