1 /**
2 	Copyright: © 2013 rejectedsoftware e.K.
3 	License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file.
4 	Authors: Sönke Ludwig
5 */
6 module dubregistry.web;
7 
8 import dubregistry.dbcontroller;
9 import dubregistry.repositories.bitbucket;
10 import dubregistry.repositories.github;
11 import dubregistry.registry;
12 import dubregistry.viewutils; // dummy import to make rdmd happy
13 
14 import dub.semver;
15 import std.algorithm : sort, startsWith;
16 import std.array;
17 import std.file;
18 import std.path;
19 import std.string;
20 import userman.web;
21 import vibe.d;
22 
23 
24 DubRegistryWebFrontend registerDubRegistryWebFrontend(URLRouter router, DubRegistry registry, UserManController userman)
25 {
26 	auto webfrontend = new DubRegistryWebFrontend(registry, userman);
27 	router.registerUserManWebInterface(userman);
28 	router.registerWebInterface(webfrontend);
29 	router.get("*", serveStaticFiles("./public"));
30 	return webfrontend;
31 }
32 
33 class DubRegistryWebFrontend {
34 	private {
35 		DubRegistry m_registry;
36 		UserManController m_userman;
37 		UserManWebAuthenticator m_usermanauth;
38 		Category[] m_categories;
39 		Category[string] m_categoryMap;
40 	}
41 
42 	this(DubRegistry registry, UserManController userman)
43 	{
44 		m_registry = registry;
45 		m_userman = userman;
46 		m_usermanauth = new UserManWebAuthenticator(userman);
47 
48 		updateCategories();
49 	}
50 
51 	// compatibility route
52 	void getAvailable(HTTPServerRequest req, HTTPServerResponse res)
53 	{
54 		res.writeJsonBody(m_registry.availablePackages.array);
55 	}
56 
57 	@path("/packages/index.json")
58 	void getPackages(HTTPServerRequest req, HTTPServerResponse res)
59 	{
60 		res.writeJsonBody(m_registry.availablePackages.array);
61 	}
62 
63 	@path("/")
64 	void getHome(string sort = "updated", string category = null)
65 	{
66 		// collect the package list
67 		auto packapp = appender!(Json[])();
68 		packapp.reserve(200);
69 		if (category.length) {
70 			foreach (pname; m_registry.availablePackages) {
71 				auto pack = m_registry.getPackageInfo(pname);
72 				foreach (c; pack.categories) {
73 					if (c.get!string.startsWith(category)) {
74 						packapp.put(pack);
75 						break;
76 					}
77 				}
78 			}
79 		} else {
80 			foreach (pack; m_registry.availablePackages)
81 				packapp.put(m_registry.getPackageInfo(pack));
82 		}
83 		auto packages = packapp.data;
84 
85 		// sort by date of last version
86 		string getDate(Json p) {
87 			if( p.type != Json.Type.Object || "versions" !in p ) return null;
88 			if( p.versions.length == 0 ) return null;
89 			return p.versions[p.versions.length-1].date.get!string;
90 		}
91 		SysTime getDateAdded(Json p) {
92 			return SysTime.fromISOExtString(p.dateAdded.get!string);
93 		}
94 		bool compare(Json a, Json b) {
95 			bool a_has_ver = a.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~"));
96 			bool b_has_ver = b.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~"));
97 			if (a_has_ver != b_has_ver) return a_has_ver;
98 			return getDate(a) > getDate(b);
99 		}
100 		switch (sort) {
101 			default: std.algorithm.sort!((a, b) => compare(a, b))(packages); break;
102 			case "name": std.algorithm.sort!((a, b) => a.name < b.name)(packages); break;
103 			case "added": std.algorithm.sort!((a, b) => getDateAdded(a) > getDateAdded(b))(packages); break;
104 		}
105 
106 		auto categories = m_categories;
107 		auto categoryMap = m_categoryMap;
108 		render!("home.dt", categories, categoryMap, packages);
109 	}
110 
111 
112 	void querySearch(string q = "")
113 	{
114 		auto results = m_registry.searchPackages(q);
115 		auto queryString = q;
116 		render!("search_results.dt", queryString, results);
117 	}
118 
119 	void getGettingStarted() { render!("getting_started.dt"); }
120 	void getAbout() { redirect("/getting_started"); }
121 	void getUsage() { redirect("/getting_started"); }
122 
123 	void getPublish() { render!("publish.dt"); }
124 	void getDevelop() { render!("develop.dt"); }
125 
126 	@path("/package-format")
127 	void getPackageFormat(string lang = null)
128 	{
129 		switch (lang) {
130 			default: redirect("package-format?lang=sdl"); break;
131 			case "json": render!("package_format_json.dt"); break;
132 			case "sdl": render!("package_format_sdl.dt"); break;
133 		}
134 	}
135 
136 	private auto downloadInfo()
137 	{
138 		static struct DownloadFile {
139 			string fileName;
140 			string platformCaption;
141 			string typeCaption;
142 		}
143 
144 		static struct DownloadVersion {
145 			string id;
146 			DownloadFile[][string] files;
147 		}
148 
149 		static struct Info {
150 			DownloadVersion[] versions;
151 			string latest = "";
152 
153 			void addFile(string ver, string platform, string filename)
154 			{
155 
156 				auto df = DownloadFile(filename);
157 				switch (platform) {
158 					default:
159 						auto pts = platform.split("-");
160 						df.platformCaption = format("%s%s (%s)", pts[0][0 .. 1].toUpper(), pts[0][1 .. $], pts[1].replace("_", "-").toUpper());
161 						break;
162 					case "osx-x86": df.platformCaption = "OS X (X86)"; break;
163 					case "osx-x86_64": df.platformCaption = "OS X (X86-64)"; break;
164 				}
165 
166 				if (filename.endsWith(".tar.gz")) df.typeCaption = "binary tarball";
167 				else if (filename.endsWith(".zip")) df.typeCaption = "zipped binaries";
168 				else if (filename.endsWith(".rpm")) df.typeCaption = "binary RPM package";
169 				else if (filename.endsWith("setup.exe")) df.typeCaption = "installer";
170 				else df.typeCaption = "Unknown";
171 
172 				foreach(ref v; versions)
173 					if( v.id == ver ){
174 						v.files[platform] ~= df;
175 						return;
176 					}
177 				DownloadVersion dv = DownloadVersion(ver);
178 				dv.files[platform] = [df];
179 				versions ~= dv;
180 				if (!isPreReleaseVersion(ver) && (latest.empty || compareVersions(ver, latest) > 0))
181 					latest = ver;
182 			}
183 		}
184 
185 		Info info;
186 
187 		if (!"public/files".exists || !"public/files".isDir)
188 			return info;
189 
190 		import std.regex;
191 		Regex!char[][string] platformPatterns;
192 		platformPatterns["windows-x86"] = [
193 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.*))?(?:-setup\\.exe|-windows-x86\\.zip)$")
194 		];
195 		platformPatterns["linux-x86_64"] = [
196 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86_64\\.tar\\.gz$"),
197 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86_64\\.rpm$")
198 		];
199 		platformPatterns["linux-x86"] = [
200 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86\\.tar\\.gz$"),
201 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86\\.rpm$")
202 		];
203 		platformPatterns["linux-arm"] = [
204 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-arm\\.tar\\.gz$"),
205 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.arm\\.rpm$")
206 		];
207 		platformPatterns["osx-x86_64"] = [
208 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-osx-x86_64\\.tar\\.gz$"),
209 		];
210 
211 		foreach(de; dirEntries("public/files", "*.*", SpanMode.shallow)) {
212 			auto name = Path(de.name).head.toString();
213 
214 			foreach (platform, rexes; platformPatterns) {
215 				foreach (rex; rexes) {
216 					auto match = match(name, rex).captures;//matchFirst(name, rex);
217 					if (match.empty) continue;
218 					auto ver = match["version"] ~ (match["prerelease"].length ? "-" ~ match["prerelease"] : "");
219 					if (!ver.isValidVersion()) continue;
220 					info.addFile(ver, platform, name);
221 				}
222 			}
223 		}
224 
225 		info.versions.sort!((a, b) => vcmp(a.id, b.id))();
226 		return info;
227 	}
228 
229 	void getDownload()
230 	{
231 		auto info = downloadInfo();
232 		render!("download.dt", info);
233 	}
234 
235 	@path("/download/LATEST")
236 	void getLatest(HTTPServerResponse res)
237 	{
238 		auto info = downloadInfo();
239 		enforceHTTP(!info.latest.empty, HTTPStatus.notFound, "No version available.");
240 		res.writeBody(info.latest);
241 	}
242 
243 	@path("/view_package/:packname")
244 	void getRedirectViewPackage(string _packname)
245 	{
246 		redirect("/packages/"~_packname);
247 	}
248 
249 	@path("/packages/:packname")
250 	void getPackage(HTTPServerResponse res, string _packname)
251 	{
252 		bool json = false;
253 		auto pname = _packname;
254 		if( pname.endsWith(".json") ){
255 			pname = pname[0 .. $-5];
256 			json = true;
257 		}
258 
259 		Json packageInfo, versionInfo;
260 		if (!getPackageInfo(pname, null, packageInfo, versionInfo))
261 			return;
262 
263 		auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string));
264 
265 		if (json) {
266 			if (pname.canFind(":")) return;
267 			res.writeJsonBody(packageInfo);
268 		} else {
269 			string packageName = pname;
270 			render!("view_package.dt", packageName, user, packageInfo, versionInfo);
271 		}
272 	}
273 
274 	@path("/packages/:packname/:version")
275 	void getPackageVersion(HTTPServerRequest req, HTTPServerResponse res, string _packname, string _version)
276 	{
277 		auto pname = _packname;
278 
279 		auto ver = req.params["version"].replace(" ", "+");
280 		string ext;
281 		if( ver.endsWith(".zip") ) ext = "zip", ver = ver[0 .. $-4];
282 		else if( ver.endsWith(".json") ) ext = "json", ver = ver[0 .. $-5];
283 
284 		Json packageInfo, versionInfo;
285 		if (!getPackageInfo(pname, ver, packageInfo, versionInfo))
286 			return;
287 
288 		auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string));
289 
290 		if (ext == "zip") {
291 			if (pname.canFind(":")) return;
292 
293 			// This log line is a weird workaround to make otherwise undefined Json fields
294 			// available. Smells like a compiler bug.
295 			logDebug("%s %s", packageInfo["id"].toString(), versionInfo["downloadUrl"].toString());
296 
297 			// add download to statistic
298 			m_registry.addDownload(BsonObjectID.fromString(packageInfo.id.get!string), ver, req.headers.get("User-agent", null));
299 			if (versionInfo["downloadUrl"].get!string.length > 0) {
300 				// redirect to hosting service specific URL
301 				redirect(versionInfo.downloadUrl.get!string);
302 			} else {
303 				// directly forward from hoster
304 				res.headers["Content-Disposition"] = "attachment; filename=\""~pname~"-"~(ver.startsWith("~") ? ver[1 .. $] : ver) ~ ".zip\"";
305 				m_registry.downloadPackageZip(pname, ver.startsWith("~") ? ver : "v"~ver, (scope data) {
306 					res.writeBody(data, "application/zip");
307 				});
308 			}
309 		} else if ( ext == "json") {
310 			if (pname.canFind(":")) return;
311 			res.writeJsonBody(versionInfo);
312 		} else {
313 			auto packageName = pname;
314 			render!("view_package.dt", packageName, user, packageInfo, versionInfo);
315 		}
316 	}
317 
318 	private bool getPackageInfo(string pack_name, string pack_version, out Json pkg_info, out Json ver_info)
319 	{
320 		auto ppath = pack_name.urlDecode().split(":");
321 
322 		pkg_info = m_registry.getPackageInfo(ppath[0]);
323 		if (pkg_info.type == Json.Type.null_) return false;
324 
325 		if (pack_version.length) {
326 			foreach (v; pkg_info.versions) {
327 				if (v["version"].get!string == pack_version) {
328 					ver_info = v;
329 					break;
330 				}
331 			}
332 			if (ver_info.type != Json.Type.Object) return false;
333 		} else {
334 			import dubregistry.viewutils;
335 			if (pkg_info.versions.length == 0) return false;
336 			ver_info = getBestVersion(pkg_info.versions);
337 		}
338 
339 		foreach (i; 1 .. ppath.length) {
340 			if ("subPackages" !in ver_info) return false;
341 			bool found = false;
342 			foreach (sp; ver_info.subPackages) {
343 				if (sp.name == ppath[i]) {
344 					Json newv = Json.emptyObject;
345 					// inherit certain fields
346 					foreach (field; ["version", "date", "license", "authors", "homepage"])
347 						if (auto pv = field in ver_info) newv[field] = *pv;
348 					// copy/overwrite the rest frmo the sub package
349 					foreach (string name, value; sp) newv[name] = value;
350 					ver_info = newv;
351 					found = true;
352 					break;
353 				}
354 			}
355 			if (!found) return false;
356 		}
357 		return true;
358 	}
359 
360 	@auth
361 	void getMyPackages(User _user)
362 	{
363 		auto user = _user;
364 		auto registry = m_registry;
365 		render!("my_packages.dt", user, registry);
366 	}
367 
368 	@auth @path("/register_package")
369 	void getRegisterPackage(User _user, string _error = null)
370 	{
371 		auto user = _user;
372 		string error = _error;
373 		auto registry = m_registry;
374 		render!("my_packages.register.dt", user, error, registry);
375 	}
376 
377 	@auth @path("/register_package") @errorDisplay!getRegisterPackage
378 	void postRegisterPackage(string kind, string owner, string project, User _user)
379 	{
380 		Json rep = Json.emptyObject;
381 		rep["kind"] = kind;
382 		rep["owner"] = owner;
383 		rep["project"] = project;
384 		m_registry.addPackage(rep, _user.id);
385 		redirect("/my_packages");
386 	}
387 
388 	@auth @path("/my_packages/:packname")
389 	void getMyPackagesPackage(string _packname, User _user, string _error = null)
390 	{
391 		enforceUserPackage(_user, _packname);
392 		auto packageName = _packname;
393 		auto nfo = m_registry.getPackageInfo(packageName);
394 		if (nfo.type == Json.Type.null_) return;
395 		auto categories = m_categories;
396 		auto registry = m_registry;
397 		auto user = _user;
398 		auto error = _error;
399 		render!("my_packages.package.dt", packageName, categories, user, registry, error);
400 	}
401 
402 	@auth @path("/my_packages/:packname/update")
403 	void postUpdatePackage(string _packname, User _user)
404 	{
405 		enforceUserPackage(_user, _packname);
406 		m_registry.triggerPackageUpdate(_packname);
407 		redirect("/my_packages/"~_packname);
408 	}
409 
410 	@auth @path("/my_packages/:packname/remove")
411 	void postShowRemovePackage(string _packname, User _user)
412 	{
413 		auto packageName = _packname;
414 		auto user = _user;
415 		enforceUserPackage(user, packageName);
416 		render!("my_packages.remove.dt", packageName, user);
417 	}
418 
419 	@auth @path("/my_packages/:packname/remove_confirm")
420 	void postRemovePackage(string _packname, User _user)
421 	{
422 		enforceUserPackage(_user, _packname);
423 		m_registry.removePackage(_packname, _user.id);
424 		redirect("/my_packages");
425 	}
426 
427 	@auth @path("/my_packages/:packname/set_categories")
428 	void postSetPackageCategories(string[] categories, string _packname, User _user)
429 	{
430 		enforceUserPackage(_user, _packname);
431 		string[] uniquecategories;
432 		outer: foreach (cat; categories) {
433 			if (!cat.length) continue;
434 			foreach (j, ec; uniquecategories) {
435 				if (cat.startsWith(ec)) continue outer;
436 				if (ec.startsWith(cat)) {
437 					uniquecategories[j] = cat;
438 					continue outer;
439 				}
440 			}
441 			uniquecategories ~= cat;
442 		}
443 		m_registry.setPackageCategories(_packname, uniquecategories);
444 
445 		redirect("/my_packages/"~_packname);
446 	}
447 
448 	@auth @path("/my_packages/:packname/set_repository") @errorDisplay!getMyPackagesPackage
449 	void postSetPackageRepository(string kind, string owner, string project, string _packname, User _user)
450 	{
451 		enforceUserPackage(_user, _packname);
452 
453 		Json rep = Json.emptyObject;
454 		rep["kind"] = kind;
455 		rep["owner"] = owner;
456 		rep["project"] = project;
457 		m_registry.setPackageRepository(_packname, rep);
458 
459 		redirect("/my_packages/"~_packname);
460 	}
461 
462 	@path("/docs/commandline")
463 	void getCommandLineDocs()
464 	{
465 		import dub.commandline;
466 		auto commands = getCommands();
467 		render!("docs.commandline.dt", commands);
468 	}
469 
470 	private void enforceUserPackage(User user, string package_name)
471 	{
472 		enforceHTTP(m_registry.isUserPackage(user.id, package_name), HTTPStatus.forbidden, "You don't have access rights for this package.");
473 	}
474 
475 	private void updateCategories()
476 	{
477 		auto catfile = openFile("categories.json");
478 		scope(exit) catfile.close();
479 		auto json = parseJsonString(catfile.readAllUTF8());
480 
481 		Category[string] catmap;
482 
483 		Category processNode(Json node, string[] path)
484 		{
485 			path ~= node.name.get!string;
486 			auto cat = new Category;
487 			cat.name = path.join(".");
488 			cat.description = node.description.get!string;
489 			if (path.length > 2)
490 				cat.indentedDescription = "\u00a0\u00a0\u00a0\u00a0".replicate(path.length-2) ~ "\u00a0└ " ~ cat.description;
491 			else if (path.length == 2)
492 				cat.indentedDescription = "\u00a0└ " ~ cat.description;
493 			else cat.indentedDescription = cat.description;
494 			foreach_reverse (i; 0 .. path.length)
495 				if (existsFile("public/images/categories/"~path[0 .. i].join(".")~".png")) {
496 					cat.imageName = path[0 .. i].join(".");
497 					break;
498 				}
499 
500 			catmap[cat.name] = cat;
501 
502 			if ("categories" in node)
503 				foreach (subcat; node.categories)
504 					cat.subCategories ~= processNode(subcat, path);
505 
506 			return cat;
507 		}
508 
509 		Category[] cats;
510 		foreach (top_level_cat; json)
511 			cats ~= processNode(top_level_cat, null);
512 
513 		m_categories = cats;
514 		m_categoryMap = catmap;
515 	}
516 
517 	// Attribute for authenticated routes
518 	private enum auth = before!performAuth("_user");
519 	mixin PrivateAccessProxy;
520 
521 	private User performAuth(HTTPServerRequest req, HTTPServerResponse res)
522 	{
523 		return m_usermanauth.performAuth(req, res);
524 	}
525 }
526 
527 final class Category {
528 	string name, description, indentedDescription, imageName;
529 	Category[] subCategories;
530 }