1 /**
2 	Copyright: © 2013 rejectedsoftware e.K.
3 	License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file.
4 	Authors: Sönke Ludwig
5 */
6 module dubregistry.web;
7 
8 import dubregistry.dbcontroller;
9 import dubregistry.repositories.bitbucket;
10 import dubregistry.repositories.github;
11 import dubregistry.registry;
12 import dubregistry.viewutils; // dummy import to make rdmd happy
13 
14 import dub.semver;
15 import std.algorithm : sort, startsWith;
16 import std.array;
17 import std.file;
18 import std.path;
19 import std.string;
20 import userman.web;
21 import vibe.d;
22 
23 
24 DubRegistryWebFrontend registerDubRegistryWebFrontend(URLRouter router, DubRegistry registry, UserManController userman)
25 {
26 	DubRegistryWebFrontend webfrontend;
27 	if (userman) {
28 		auto ff = new DubRegistryFullWebFrontend(registry, userman);
29 		webfrontend = ff;
30 		router.registerWebInterface(ff);
31 		router.registerUserManWebInterface(userman);
32 	} else {
33 		webfrontend = new DubRegistryWebFrontend(registry, userman);
34 		router.registerWebInterface(webfrontend);
35 	}
36 	router.get("*", serveStaticFiles("./public"));
37 	return webfrontend;
38 }
39 
40 class DubRegistryWebFrontend {
41 	protected {
42 		DubRegistry m_registry;
43 		UserManController m_userman;
44 		Category[] m_categories;
45 		Category[string] m_categoryMap;
46 		DbPackage[] m_packages;
47 	}
48 
49 	this(DubRegistry registry, UserManController userman)
50 	{
51 		m_registry = registry;
52 		m_userman = userman;
53 		updateCategories();
54 		updatePackageList();
55 	}
56 
57 	@path("/")
58 	void getHome(string sort = "updated", string category = null, ulong skip = 0, ulong limit = 20)
59 	{
60 		import std.algorithm.comparison : min;
61 		import std.algorithm.iteration : filter, map;
62 
63 		static struct Info {
64 			Json[] packages;
65 			size_t packageCount;
66 			size_t skip;
67 			size_t limit;
68 			Category[] categories;
69 			Category[string] categoryMap;
70 		}
71 
72 		static import std.algorithm.sorting;
73 		import std.algorithm.searching : any;
74 
75 		DbPackage[] packages;
76 		if (category.length) {
77 			packages = m_packages
78 				.filter!(p => p.categories.any!(c => c.startsWith(category)))
79 				.array;
80 		} else {
81 			packages = m_packages.dup;
82 		}
83 
84 		// sort by date of last version
85 		SysTime getDate(in ref DbPackage p) {
86 			if (p.versions.length == 0) return SysTime(0, UTC());
87 			return p.versions[$-1].date;
88 		}
89 		SysTime getDateAdded(in ref DbPackage p) { return (cast(BsonObjectID*)&p._id).timeStamp; }
90 		bool compare(in ref DbPackage a, in ref DbPackage b) {
91 			bool a_has_ver = a.versions.any!(v => !v.version_.startsWith("~"));
92 			bool b_has_ver = b.versions.any!(v => !v.version_.startsWith("~"));
93 			if (a_has_ver != b_has_ver) return a_has_ver;
94 			return getDate(a) > getDate(b);
95 		}
96 		switch (sort) {
97 			default: std.algorithm.sorting.sort!compare(packages); break;
98 			case "name": std.algorithm.sorting.sort!((a, b) => a.name < b.name)(packages); break;
99 			case "added": std.algorithm.sorting.sort!((a, b) => getDateAdded(a) > getDateAdded(b))(packages); break;
100 		}
101 
102 		Info info;
103 		info.packageCount = packages.length;
104 		info.packages = packages[min(skip, $) .. min(skip + limit, $)]
105 			.map!(p => m_registry.getPackageInfo(p, false).info)
106 			.array;
107 		info.skip = skip;
108 		info.limit = limit;
109 		info.categories = m_categories;
110 		info.categoryMap = m_categoryMap;
111 
112 		render!("home.dt", info);
113 	}
114 
115 	// compatibility route
116 	void getAvailable(HTTPServerRequest req, HTTPServerResponse res)
117 	{
118 		res.redirect("/packages/index.json");
119 	}
120 
121 	@path("/packages/index.json")
122 	void getPackages(HTTPServerRequest req, HTTPServerResponse res)
123 	{
124 		res.writeJsonBody(m_registry.availablePackages.array);
125 	}
126 
127 	@path("/view_package/:packname")
128 	void getRedirectViewPackage(string _packname)
129 	{
130 		redirect("/packages/"~_packname);
131 	}
132 
133 	@path("/packages/:packname")
134 	void getPackage(HTTPServerRequest req, HTTPServerResponse res, string _packname)
135 	{
136 		getPackageVersion(req, res, _packname, null);
137 	}
138 
139 	@path("/packages/:packname/:version")
140 	void getPackageVersion(HTTPServerRequest req, HTTPServerResponse res, string _packname, string _version)
141 	{
142 		import std.algorithm : canFind;
143 
144 		auto pname = _packname;
145 		auto ver = _version.replace(" ", "+");
146 		string ext;
147 
148 		if (_version.length) {
149 			if (ver.endsWith(".zip")) ext = "zip", ver = ver[0 .. $-4];
150 			else if( ver.endsWith(".json") ) ext = "json", ver = ver[0 .. $-5];
151 		} else {
152 			if (pname.endsWith(".json")) {
153 				pname = pname[0 .. $-5];
154 				ext = "json";
155 			}
156 		}
157 
158 		PackageInfo packinfo;
159 		PackageVersionInfo verinfo;
160 		if (!getPackageInfo(pname, ver, packinfo, verinfo))
161 			return;
162 
163 		auto packageInfo = packinfo.info;
164 		auto versionInfo = verinfo.info;
165 
166 		User user;
167 		if (m_userman) {
168 			try user = m_userman.getUser(User.ID.fromString(packageInfo["owner"].get!string));
169 			catch (Exception e) {
170 				logDebug("Failed to get owner '%s' for %s %s: %s",
171 					packageInfo["owner"].get!string, pname, ver, e.msg);
172 			}
173 		}
174 
175 		if (ext == "zip") {
176 			if (pname.canFind(":")) return;
177 
178 			// This log line is a weird workaround to make otherwise undefined Json fields
179 			// available. Smells like a compiler bug.
180 			logDebug("%s %s", packageInfo["id"].toString(), verinfo.downloadURL);
181 
182 			// add download to statistic
183 			m_registry.addDownload(BsonObjectID.fromString(packageInfo["id"].get!string), ver, req.headers.get("User-agent", null));
184 			if (verinfo.downloadURL.length > 0) {
185 				// redirect to hosting service specific URL
186 				redirect(verinfo.downloadURL);
187 			} else {
188 				// directly forward from hoster
189 				res.headers["Content-Disposition"] = "attachment; filename=\""~pname~"-"~(ver.startsWith("~") ? ver[1 .. $] : ver) ~ ".zip\"";
190 				m_registry.downloadPackageZip(pname, ver.startsWith("~") ? ver : "v"~ver, (scope data) {
191 					res.writeBody(data, "application/zip");
192 				});
193 			}
194 		} else if (ext == "json") {
195 			if (pname.canFind(":")) return;
196 			res.writeJsonBody(_version.length ? versionInfo : packageInfo);
197 		} else {
198 			auto gitVer = verinfo.version_;
199 			gitVer = gitVer.startsWith("~") ? gitVer[1 .. $] : "v"~gitVer;
200 			string urlFilter(string url, bool is_image)
201 			{
202 				if (url.startsWith("http://") || url.startsWith("https://"))
203 					return url;
204 
205 				if (auto pr = "repository" in packageInfo) {
206 					auto owner = (*pr)["owner"].get!string;
207 					auto project = (*pr)["project"].get!string;
208 					switch ((*pr)["kind"].get!string) {
209 						default: return url;
210 						// TODO: BitBucket + GitLab
211 						case "github":
212 							if (is_image) return format("https://github.com/%s/%s/raw/%s/%s", owner, project, gitVer, url);
213 							else return format("https://github.com/%s/%s/blob/%s/%s", owner, project, gitVer, url);
214 					}
215 				}
216 
217 				return url;
218 			}
219 
220 			auto packageName = pname;
221 			auto registry = m_registry;
222 			auto readmeContents = m_registry.getReadme(versionInfo, packageInfo["repository"].deserializeJson!DbRepository);
223 			render!("view_package.dt", packageName, user, packageInfo, versionInfo, readmeContents, urlFilter, registry);
224 		}
225 	}
226 
227 	@path("/packages/:packname/versions")
228 	void getAllPackageVersions(HTTPServerRequest req, HTTPServerResponse res, string _packname) {
229 		import std.algorithm : canFind;
230 
231 		auto pname = _packname;
232 
233 		auto ppath = _packname.urlDecode().split(":");
234 		auto packageInfo = m_registry.getPackageInfo(ppath[0]);
235 
236 		auto packageName = pname;
237 		auto registry = m_registry;
238 		render!("view_package.versions.dt",
239 			packageName,
240 			packageInfo,
241 			registry);
242 	}
243 
244 	private bool getPackageInfo(string pack_name, string pack_version, out PackageInfo pkg_info, out PackageVersionInfo ver_info) {
245 		import std.algorithm : map;
246 		auto ppath = pack_name.urlDecode().split(":");
247 
248 		pkg_info = m_registry.getPackageInfo(ppath[0]);
249 		if (pkg_info.info.type == Json.Type.null_) return false;
250 
251 		if (pack_version.length) {
252 			foreach (ref v; pkg_info.versions) {
253 				if (v.version_ == pack_version) {
254 					ver_info = v;
255 					break;
256 				}
257 			}
258 			if (ver_info.info.type != Json.Type.Object) return false;
259 		} else {
260 			import dubregistry.viewutils;
261 			if (pkg_info.versions.length == 0) return false;
262 			auto vidx = getBestVersionIndex(pkg_info.versions.map!(v => v.version_));
263 			ver_info = pkg_info.versions[vidx];
264 		}
265 
266 		foreach (i; 1 .. ppath.length) {
267 			if ("subPackages" !in ver_info.info) return false;
268 			bool found = false;
269 			foreach (sp; ver_info.info["subPackages"]) {
270 				if (sp["name"] == ppath[i]) {
271 					Json newv = Json.emptyObject;
272 					// inherit certain fields
273 					foreach (field; ["version", "date", "license", "authors", "homepage"])
274 						if (auto pv = field in ver_info.info) newv[field] = *pv;
275 					// copy/overwrite the rest frmo the sub package
276 					foreach (string name, value; sp) newv[name] = value;
277 					ver_info.info = newv;
278 					found = true;
279 					break;
280 				}
281 			}
282 			if (!found) return false;
283 		}
284 		return true;
285 	}
286 
287 	private void updateCategories()
288 	{
289 		auto catfile = openFile("categories.json");
290 		scope(exit) catfile.close();
291 		auto json = parseJsonString(catfile.readAllUTF8());
292 
293 		Category[string] catmap;
294 
295 		Category processNode(Json node, string[] path)
296 		{
297 			path ~= node["name"].get!string;
298 			auto cat = new Category;
299 			cat.name = path.join(".");
300 			cat.description = node["description"].get!string;
301 			if (path.length > 2)
302 				cat.indentedDescription = "\u00a0\u00a0\u00a0\u00a0".replicate(path.length-2) ~ "\u00a0└ " ~ cat.description;
303 			else if (path.length == 2)
304 				cat.indentedDescription = "\u00a0└ " ~ cat.description;
305 			else cat.indentedDescription = cat.description;
306 			foreach_reverse (i; 0 .. path.length)
307 				if (existsFile("public/images/categories/"~path[0 .. i+1].join(".")~".png")) {
308 					cat.imageName = path[0 .. i+1].join(".");
309 					break;
310 				}
311 
312 			catmap[cat.name] = cat;
313 
314 			if ("categories" in node)
315 				foreach (subcat; node["categories"])
316 					cat.subCategories ~= processNode(subcat, path);
317 
318 			return cat;
319 		}
320 
321 		Category[] cats;
322 		foreach (top_level_cat; json)
323 			cats ~= processNode(top_level_cat, null);
324 
325 		m_categories = cats;
326 		m_categoryMap = catmap;
327 	}
328 
329 	private void updatePackageList()
330 	{
331 		setTimer(30.seconds, &updatePackageList);
332 		m_packages = m_registry.getPackageDump().array;
333 	}
334 }
335 
336 class DubRegistryFullWebFrontend : DubRegistryWebFrontend {
337 	private {
338 		UserManWebAuthenticator m_usermanauth;
339 	}
340 
341 	this(DubRegistry registry, UserManController userman)
342 	{
343 		super(registry, userman);
344 		m_usermanauth = new UserManWebAuthenticator(userman);
345 	}
346 
347 	void querySearch(string q = "")
348 	{
349 		auto results = m_registry.searchPackages(q);
350 		auto queryString = q;
351 		render!("search_results.dt", queryString, results);
352 	}
353 
354 	void getGettingStarted() { render!("getting_started.dt"); }
355 	void getAbout() { redirect("/getting_started"); }
356 	void getUsage() { redirect("/getting_started"); }
357 
358 	void getPublish() { render!("publish.dt"); }
359 	void getDevelop() { render!("develop.dt"); }
360 
361 	@path("/package-format")
362 	void getPackageFormat(string lang = null)
363 	{
364 		switch (lang) {
365 			default: redirect("package-format?lang=json"); break;
366 			case "json": render!("package_format_json.dt"); break;
367 			case "sdl": render!("package_format_sdl.dt"); break;
368 		}
369 	}
370 
371 	private auto downloadInfo()
372 	{
373 		static struct DownloadFile {
374 			string fileName;
375 			string platformCaption;
376 			string typeCaption;
377 		}
378 
379 		static struct DownloadVersion {
380 			string id;
381 			DownloadFile[][string] files;
382 		}
383 
384 		static struct Info {
385 			DownloadVersion[] versions;
386 			string latest = "";
387 
388 			void addFile(string ver, string platform, string filename)
389 			{
390 
391 				auto df = DownloadFile(filename);
392 				switch (platform) {
393 					default:
394 						auto pts = platform.split("-");
395 						df.platformCaption = format("%s%s (%s)", pts[0][0 .. 1].toUpper(), pts[0][1 .. $], pts[1].replace("_", "-").toUpper());
396 						break;
397 					case "osx-x86": df.platformCaption = "OS X (X86)"; break;
398 					case "osx-x86_64": df.platformCaption = "OS X (X86-64)"; break;
399 				}
400 
401 				if (filename.endsWith(".tar.gz")) df.typeCaption = "binary tarball";
402 				else if (filename.endsWith(".zip")) df.typeCaption = "zipped binaries";
403 				else if (filename.endsWith(".rpm")) df.typeCaption = "binary RPM package";
404 				else if (filename.endsWith("setup.exe")) df.typeCaption = "installer";
405 				else df.typeCaption = "Unknown";
406 
407 				foreach(ref v; versions)
408 					if( v.id == ver ){
409 						v.files[platform] ~= df;
410 						return;
411 					}
412 				DownloadVersion dv = DownloadVersion(ver);
413 				dv.files[platform] = [df];
414 				versions ~= dv;
415 				if (!isPreReleaseVersion(ver) && (latest.empty || compareVersions(ver, latest) > 0))
416 					latest = ver;
417 			}
418 		}
419 
420 		Info info;
421 
422 		if (!"public/files".exists || !"public/files".isDir)
423 			return info;
424 
425 		import std.regex;
426 		Regex!char[][string] platformPatterns;
427 		platformPatterns["windows-x86"] = [
428 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.*))?(?:-setup\\.exe|-windows-x86\\.zip)$")
429 		];
430 		platformPatterns["linux-x86_64"] = [
431 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86_64\\.tar\\.gz$"),
432 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86_64\\.rpm$")
433 		];
434 		platformPatterns["linux-x86"] = [
435 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86\\.tar\\.gz$"),
436 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86\\.rpm$")
437 		];
438 		platformPatterns["linux-arm"] = [
439 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-arm\\.tar\\.gz$"),
440 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.arm\\.rpm$")
441 		];
442 		platformPatterns["osx-x86_64"] = [
443 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-osx-x86_64\\.tar\\.gz$"),
444 		];
445 
446 		foreach(de; dirEntries("public/files", "*.*", SpanMode.shallow)) {
447 			auto name = Path(de.name).head.toString();
448 
449 			foreach (platform, rexes; platformPatterns) {
450 				foreach (rex; rexes) {
451 					auto match = match(name, rex).captures;//matchFirst(name, rex);
452 					if (match.empty) continue;
453 					auto ver = match["version"] ~ (match["prerelease"].length ? "-" ~ match["prerelease"] : "");
454 					if (!ver.isValidVersion()) continue;
455 					info.addFile(ver, platform, name);
456 				}
457 			}
458 		}
459 
460 		info.versions.sort!((a, b) => vcmp(a.id, b.id))();
461 		return info;
462 	}
463 
464 	void getDownload()
465 	{
466 		auto info = downloadInfo();
467 		render!("download.dt", info);
468 	}
469 
470 	@path("/download/LATEST")
471 	void getLatest(HTTPServerResponse res)
472 	{
473 		auto info = downloadInfo();
474 		enforceHTTP(!info.latest.empty, HTTPStatus.notFound, "No version available.");
475 		res.writeBody(info.latest);
476 	}
477 
478 
479 	@auth
480 	void getMyPackages(User _user)
481 	{
482 		auto user = _user;
483 		auto registry = m_registry;
484 		render!("my_packages.dt", user, registry);
485 	}
486 
487 	@auth @path("/register_package")
488 	void getRegisterPackage(User _user, string kind = null, string owner = null, string project = null, string _error = null)
489 	{
490 		auto user = _user;
491 		string error = _error;
492 		auto registry = m_registry;
493 		render!("my_packages.register.dt", user, kind, owner, project, error, registry);
494 	}
495 
496 	@auth @path("/register_package") @errorDisplay!getRegisterPackage
497 	void postRegisterPackage(string kind, string owner, string project, User _user, bool ignore_fork = false)
498 	{
499 		DbRepository rep;
500 		rep.kind = kind;
501 		rep.owner = owner;
502 		rep.project = project;
503 
504 		if (!ignore_fork) {
505 			auto info = m_registry.getRepositoryInfo(rep);
506 			if (info.isFork) {
507 				render!("my_packages.register.warn_fork.dt", kind, owner, project);
508 				return;
509 			}
510 		}
511 
512 		m_registry.addPackage(rep, _user.id);
513 		redirect("/my_packages");
514 	}
515 
516 	@auth @path("/my_packages/:packname")
517 	void getMyPackagesPackage(string _packname, User _user, string _error = null)
518 	{
519 		enforceUserPackage(_user, _packname);
520 		auto packageName = _packname;
521 		auto nfo = m_registry.getPackageInfo(packageName);
522 		if (nfo.info.type == Json.Type.null_) return;
523 		auto categories = m_categories;
524 		auto registry = m_registry;
525 		auto user = _user;
526 		auto error = _error;
527 		render!("my_packages.package.dt", packageName, categories, user, registry, error);
528 	}
529 
530 	@auth @path("/my_packages/:packname/update")
531 	void postUpdatePackage(string _packname, User _user)
532 	{
533 		enforceUserPackage(_user, _packname);
534 		m_registry.triggerPackageUpdate(_packname);
535 		redirect("/my_packages/"~_packname);
536 	}
537 
538 	@auth @path("/my_packages/:packname/remove")
539 	void postShowRemovePackage(string _packname, User _user)
540 	{
541 		auto packageName = _packname;
542 		auto user = _user;
543 		enforceUserPackage(user, packageName);
544 		render!("my_packages.remove.dt", packageName, user);
545 	}
546 
547 	@auth @path("/my_packages/:packname/remove_confirm")
548 	void postRemovePackage(string _packname, User _user)
549 	{
550 		enforceUserPackage(_user, _packname);
551 		m_registry.removePackage(_packname, _user.id);
552 		redirect("/my_packages");
553 	}
554 
555 	@auth @path("/my_packages/:packname/set_categories")
556 	void postSetPackageCategories(string[] categories, string _packname, User _user)
557 	{
558 		enforceUserPackage(_user, _packname);
559 		string[] uniquecategories;
560 		outer: foreach (cat; categories) {
561 			if (!cat.length) continue;
562 			foreach (j, ec; uniquecategories) {
563 				if (cat.startsWith(ec)) continue outer;
564 				if (ec.startsWith(cat)) {
565 					uniquecategories[j] = cat;
566 					continue outer;
567 				}
568 			}
569 			uniquecategories ~= cat;
570 		}
571 		m_registry.setPackageCategories(_packname, uniquecategories);
572 
573 		redirect("/my_packages/"~_packname);
574 	}
575 
576 	@auth @path("/my_packages/:packname/set_repository") @errorDisplay!getMyPackagesPackage
577 	void postSetPackageRepository(string kind, string owner, string project, string _packname, User _user)
578 	{
579 		enforceUserPackage(_user, _packname);
580 
581 		DbRepository rep;
582 		rep.kind = kind;
583 		rep.owner = owner;
584 		rep.project = project;
585 		m_registry.setPackageRepository(_packname, rep);
586 
587 		redirect("/my_packages/"~_packname);
588 	}
589 
590 	@path("/docs/commandline")
591 	void getCommandLineDocs()
592 	{
593 		import dub.commandline;
594 		auto commands = getCommands();
595 		render!("docs.commandline.dt", commands);
596 	}
597 
598 	private void enforceUserPackage(User user, string package_name)
599 	{
600 		enforceHTTP(m_registry.isUserPackage(user.id, package_name), HTTPStatus.forbidden, "You don't have access rights for this package.");
601 	}
602 
603 	// Attribute for authenticated routes
604 	private enum auth = before!performAuth("_user");
605 	mixin PrivateAccessProxy;
606 
607 	private User performAuth(HTTPServerRequest req, HTTPServerResponse res)
608 	{
609 		return m_usermanauth.performAuth(req, res);
610 	}
611 }
612 
613 final class Category {
614 	string name, description, indentedDescription, imageName;
615 	Category[] subCategories;
616 }