1 /**
2 	Copyright: © 2013 rejectedsoftware e.K.
3 	License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file.
4 	Authors: Sönke Ludwig
5 */
6 module dubregistry.web;
7 
8 import dubregistry.dbcontroller;
9 import dubregistry.repositories.bitbucket;
10 import dubregistry.repositories.github;
11 import dubregistry.registry;
12 import dubregistry.viewutils; // dummy import to make rdmd happy
13 
14 import dub.semver;
15 import std.algorithm : sort, startsWith;
16 import std.array;
17 import std.file;
18 import std.path;
19 import std.string;
20 import userman.web;
21 import vibe.d;
22 
23 
24 DubRegistryWebFrontend registerDubRegistryWebFrontend(URLRouter router, DubRegistry registry, UserManController userman)
25 {
26 	auto webfrontend = new DubRegistryWebFrontend(registry, userman);
27 	router.registerUserManWebInterface(userman);
28 	router.registerWebInterface(webfrontend);
29 	router.get("*", serveStaticFiles("./public"));
30 	return webfrontend;
31 }
32 
33 class DubRegistryWebFrontend {
34 	private {
35 		DubRegistry m_registry;
36 		UserManController m_userman;
37 		UserManWebAuthenticator m_usermanauth;
38 		Category[] m_categories;
39 		Category[string] m_categoryMap;
40 	}
41 
42 	this(DubRegistry registry, UserManController userman)
43 	{
44 		m_registry = registry;
45 		m_userman = userman;
46 		m_usermanauth = new UserManWebAuthenticator(userman);
47 
48 		updateCategories();
49 	}
50 
51 	// compatibility route
52 	void getAvailable(HTTPServerRequest req, HTTPServerResponse res)
53 	{
54 		res.writeJsonBody(m_registry.availablePackages.array);
55 	}
56 
57 	@path("/packages/index.json")
58 	void getPackages(HTTPServerRequest req, HTTPServerResponse res)
59 	{
60 		res.writeJsonBody(m_registry.availablePackages.array);
61 	}
62 
63 	@path("/")
64 	void getHome(string sort = "updated", string category = null)
65 	{
66 		// collect the package list
67 		auto packapp = appender!(Json[])();
68 		packapp.reserve(200);
69 		if (category.length) {
70 			foreach (pname; m_registry.availablePackages) {
71 				auto pack = m_registry.getPackageInfo(pname);
72 				foreach (c; pack.categories) {
73 					if (c.get!string.startsWith(category)) {
74 						packapp.put(pack);
75 						break;
76 					}
77 				}
78 			}
79 		} else {
80 			foreach (pack; m_registry.availablePackages)
81 				packapp.put(m_registry.getPackageInfo(pack));
82 		}
83 		auto packages = packapp.data;
84 
85 		// sort by date of last version
86 		string getDate(Json p) {
87 			if( p.type != Json.Type.Object || "versions" !in p ) return null;
88 			if( p.versions.length == 0 ) return null;
89 			return p.versions[p.versions.length-1].date.get!string;
90 		}
91 		SysTime getDateAdded(Json p) {
92 			return SysTime.fromISOExtString(p.dateAdded.get!string);
93 		}
94 		bool compare(Json a, Json b) {
95 			bool a_has_ver = a.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~"));
96 			bool b_has_ver = b.versions.get!(Json[]).any!(v => !v["version"].get!string.startsWith("~"));
97 			if (a_has_ver != b_has_ver) return a_has_ver;
98 			return getDate(a) > getDate(b);
99 		}
100 		switch (sort) {
101 			default: std.algorithm.sort!((a, b) => compare(a, b))(packages); break;
102 			case "name": std.algorithm.sort!((a, b) => a.name < b.name)(packages); break;
103 			case "added": std.algorithm.sort!((a, b) => getDateAdded(a) > getDateAdded(b))(packages); break;
104 		}
105 
106 		auto categories = m_categories;
107 		auto categoryMap = m_categoryMap;
108 		render!("home.dt", categories, categoryMap, packages);
109 	}
110 
111 
112 	void querySearch(string q = "")
113 	{
114 		auto queryString = q;
115 		auto keywords = queryString.split();
116 		auto results = m_registry.searchPackages(keywords);
117 		render!("search_results.dt", queryString, results);
118 	}
119 
120 	void getGettingStarted() { render!("getting_started.dt"); }
121 	void getAbout() { redirect("/getting_started"); }
122 	void getUsage() { redirect("/getting_started"); }
123 
124 	void getPublish() { render!("publish.dt"); }
125 	void getDevelop() { render!("develop.dt"); }
126 	@path("/package-format")
127 	void getPackageFormat() { render!("package_format.dt"); }
128 
129 	private auto downloadInfo()
130 	{
131 		static struct DownloadFile {
132 			string fileName;
133 			string platformCaption;
134 			string typeCaption;
135 		}
136 
137 		static struct DownloadVersion {
138 			string id;
139 			DownloadFile[][string] files;
140 		}
141 
142 		static struct Info {
143 			DownloadVersion[] versions;
144 			string latest = "";
145 
146 			void addFile(string ver, string platform, string filename)
147 			{
148 
149 				auto df = DownloadFile(filename);
150 				switch (platform) {
151 					default:
152 						auto pts = platform.split("-");
153 						df.platformCaption = format("%s%s (%s)", pts[0][0 .. 1].toUpper(), pts[0][1 .. $], pts[1].replace("_", "-").toUpper());
154 						break;
155 					case "osx-x86": df.platformCaption = "OS X (X86)"; break;
156 					case "osx-x86_64": df.platformCaption = "OS X (X86-64)"; break;
157 				}
158 
159 				if (filename.endsWith(".tar.gz")) df.typeCaption = "binary tarball";
160 				else if (filename.endsWith(".zip")) df.typeCaption = "zipped binaries";
161 				else if (filename.endsWith(".rpm")) df.typeCaption = "binary RPM package";
162 				else if (filename.endsWith("setup.exe")) df.typeCaption = "installer";
163 				else df.typeCaption = "Unknown";
164 
165 				foreach(ref v; versions)
166 					if( v.id == ver ){
167 						v.files[platform] ~= df;
168 						return;
169 					}
170 				DownloadVersion dv = DownloadVersion(ver);
171 				dv.files[platform] = [df];
172 				versions ~= dv;
173 				if (!isPreReleaseVersion(ver) && (latest.empty || compareVersions(ver, latest) > 0))
174 					latest = ver;
175 			}
176 		}
177 
178 		Info info;
179 
180 		if (!"public/files".exists || !"public/files".isDir)
181 			return info;
182 
183 		import std.regex;
184 		Regex!char[][string] platformPatterns;
185 		platformPatterns["windows-x86"] = [
186 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.*))?(?:-setup\\.exe|-windows-x86\\.zip)$")
187 		];
188 		platformPatterns["linux-x86_64"] = [
189 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86_64\\.tar\\.gz$"),
190 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86_64\\.rpm$")
191 		];
192 		platformPatterns["linux-x86"] = [
193 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-linux-x86\\.tar\\.gz$"),
194 			regex("^dub-(?P<version>[^-]+)-(?:0\\.(?P<prerelease>.+)|[^0].*)\\.x86\\.rpm$")
195 		];
196 		platformPatterns["osx-x86_64"] = [
197 			regex("^dub-(?P<version>[^-]+)(?:-(?P<prerelease>.+))?-osx-x86_64\\.tar\\.gz$"),
198 		];
199 
200 		foreach(de; dirEntries("public/files", "*.*", SpanMode.shallow)) {
201 			auto name = Path(de.name).head.toString();
202 
203 			foreach (platform, rexes; platformPatterns) {
204 				foreach (rex; rexes) {
205 					auto match = match(name, rex).captures;//matchFirst(name, rex);
206 					if (match.empty) continue;
207 					auto ver = match["version"] ~ (match["prerelease"].length ? "-" ~ match["prerelease"] : "");
208 					if (!ver.isValidVersion()) continue;
209 					info.addFile(ver, platform, name);
210 				}
211 			}
212 		}
213 
214 		info.versions.sort!((a, b) => vcmp(a.id, b.id))();
215 		return info;
216 	}
217 
218 	void getDownload()
219 	{
220 		auto info = downloadInfo();
221 		render!("download.dt", info);
222 	}
223 
224 	@path("/download/LATEST")
225 	void getLatest(HTTPServerResponse res)
226 	{
227 		auto info = downloadInfo();
228 		enforceHTTP(!info.latest.empty, HTTPStatus.notFound, "No version available.");
229 		res.writeBody(info.latest);
230 	}
231 
232 	@path("/view_package/:packname")
233 	void getRedirectViewPackage(string _packname)
234 	{
235 		redirect("/packages/"~_packname);
236 	}
237 
238 	@path("/packages/:packname")
239 	void getPackage(HTTPServerResponse res, string _packname)
240 	{
241 		bool json = false;
242 		auto pname = _packname;
243 		if( pname.endsWith(".json") ){
244 			pname = pname[0 .. $-5];
245 			json = true;
246 		}
247 
248 		Json packageInfo, versionInfo;
249 		if (!getPackageInfo(pname, null, packageInfo, versionInfo))
250 			return;
251 
252 		auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string));
253 
254 		if (json) {
255 			if (pname.canFind(":")) return;
256 			res.writeJsonBody(packageInfo);
257 		} else {
258 			string packageName = pname;
259 			render!("view_package.dt", packageName, user, packageInfo, versionInfo);
260 		}
261 	}
262 
263 	@path("/packages/:packname/:version")
264 	void getPackageVersion(HTTPServerRequest req, HTTPServerResponse res, string _packname, string _version)
265 	{
266 		auto pname = _packname;
267 
268 		auto ver = req.params["version"].replace(" ", "+");
269 		string ext;
270 		if( ver.endsWith(".zip") ) ext = "zip", ver = ver[0 .. $-4];
271 		else if( ver.endsWith(".json") ) ext = "json", ver = ver[0 .. $-5];
272 
273 		Json packageInfo, versionInfo;
274 		if (!getPackageInfo(pname, ver, packageInfo, versionInfo))
275 			return;
276 
277 		auto user = m_userman.getUser(User.ID.fromString(packageInfo.owner.get!string));
278 
279 		if (ext == "zip") {
280 			if (pname.canFind(":")) return;
281 			// add download to statistic
282 			m_registry.addDownload(BsonObjectID.fromString(packageInfo.id.get!string), ver, req.headers.get("User-agent", null));
283 			// redirect to hosting service specific URL
284 			redirect(versionInfo.downloadUrl.get!string);
285 		} else if ( ext == "json") {
286 			if (pname.canFind(":")) return;
287 			res.writeJsonBody(versionInfo);
288 		} else {
289 			auto packageName = pname;
290 			render!("view_package.dt", packageName, user, packageInfo, versionInfo);
291 		}
292 	}
293 
294 	private bool getPackageInfo(string pack_name, string pack_version, out Json pkg_info, out Json ver_info)
295 	{
296 		auto ppath = pack_name.urlDecode().split(":");
297 
298 		pkg_info = m_registry.getPackageInfo(ppath[0]);
299 		if (pkg_info == null) return false;
300 
301 		if (pack_version.length) {
302 			foreach (v; pkg_info.versions) {
303 				if (v["version"].get!string == pack_version) {
304 					ver_info = v;
305 					break;
306 				}
307 			}
308 			if (ver_info.type != Json.Type.Object) return false;
309 		} else {
310 			import dubregistry.viewutils;
311 			if (pkg_info.versions.length == 0) return false;
312 			ver_info = getBestVersion(pkg_info.versions);
313 		}
314 
315 		foreach (i; 1 .. ppath.length) {
316 			if ("subPackages" !in ver_info) return false;
317 			bool found = false;
318 			foreach (sp; ver_info.subPackages) {
319 				if (sp.name == ppath[i]) {
320 					Json newv = Json.emptyObject;
321 					// inherit certain fields
322 					foreach (field; ["version", "date", "license", "authors", "homepage"])
323 						if (auto pv = field in ver_info) newv[field] = *pv;
324 					// copy/overwrite the rest frmo the sub package
325 					foreach (string name, value; sp) newv[name] = value;
326 					ver_info = newv;
327 					found = true;
328 					break;
329 				}
330 			}
331 			if (!found) return false;
332 		}
333 		return true;
334 	}
335 
336 	@auth
337 	void getMyPackages(User _user)
338 	{
339 		auto user = _user;
340 		auto registry = m_registry;
341 		render!("my_packages.dt", user, registry);
342 	}
343 
344 	@auth @path("/register_package")
345 	void getRegisterPackage(User _user, string _error = null)
346 	{
347 		auto user = _user;
348 		string error = _error;
349 		auto registry = m_registry;
350 		render!("my_packages.register.dt", user, error, registry);
351 	}
352 
353 	@auth @path("/register_package") @errorDisplay!getRegisterPackage
354 	void postRegisterPackage(string kind, string owner, string project, User _user)
355 	{
356 		Json rep = Json.emptyObject;
357 		rep["kind"] = kind;
358 		rep["owner"] = owner;
359 		rep["project"] = project;
360 		m_registry.addPackage(rep, _user.id);
361 		redirect("/my_packages");
362 	}
363 
364 	@auth @path("/my_packages/:packname")
365 	void getMyPackagesPackage(string _packname, User _user, string _error = null)
366 	{
367 		enforceUserPackage(_user, _packname);
368 		auto packageName = _packname;
369 		auto nfo = m_registry.getPackageInfo(packageName);
370 		if (nfo.type == Json.Type.null_) return;
371 		auto categories = m_categories;
372 		auto registry = m_registry;
373 		auto user = _user;
374 		auto error = _error;
375 		render!("my_packages.package.dt", packageName, categories, user, registry, error);
376 	}
377 
378 	@auth @path("/my_packages/:packname/update")
379 	void postUpdatePackage(string _packname, User _user)
380 	{
381 		enforceUserPackage(_user, _packname);
382 		m_registry.triggerPackageUpdate(_packname);
383 		redirect("/my_packages/"~_packname);
384 	}
385 
386 	@auth @path("/my_packages/:packname/remove")
387 	void postShowRemovePackage(string _packname, User _user)
388 	{
389 		auto packageName = _packname;
390 		auto user = _user;
391 		enforceUserPackage(user, packageName);
392 		render!("my_packages.remove.dt", packageName, user);
393 	}
394 
395 	@auth @path("/my_packages/:packname/remove_confirm")
396 	void postRemovePackage(string _packname, User _user)
397 	{
398 		enforceUserPackage(_user, _packname);
399 		m_registry.removePackage(_packname, _user.id);
400 		redirect("/my_packages");
401 	}
402 
403 	@auth @path("/my_packages/:packname/set_categories")
404 	void postSetPackageCategories(string[] categories, string _packname, User _user)
405 	{
406 		enforceUserPackage(_user, _packname);
407 		string[] uniquecategories;
408 		outer: foreach (cat; categories) {
409 			if (!cat.length) continue;
410 			foreach (j, ec; uniquecategories) {
411 				if (cat.startsWith(ec)) continue outer;
412 				if (ec.startsWith(cat)) {
413 					uniquecategories[j] = cat;
414 					continue outer;
415 				}
416 			}
417 			uniquecategories ~= cat;
418 		}
419 		m_registry.setPackageCategories(_packname, uniquecategories);
420 
421 		redirect("/my_packages/"~_packname);
422 	}
423 
424 	@auth @path("/my_packages/:packname/set_repository") @errorDisplay!getMyPackagesPackage
425 	void postSetPackageRepository(string kind, string owner, string project, string _packname, User _user)
426 	{
427 		enforceUserPackage(_user, _packname);
428 
429 		Json rep = Json.emptyObject;
430 		rep["kind"] = kind;
431 		rep["owner"] = owner;
432 		rep["project"] = project;
433 		m_registry.setPackageRepository(_packname, rep);
434 
435 		redirect("/my_packages/"~_packname);
436 	}
437 
438 	private void enforceUserPackage(User user, string package_name)
439 	{
440 		enforceHTTP(m_registry.isUserPackage(user.id, package_name), HTTPStatus.forbidden, "You don't have access rights for this package.");
441 	}
442 
443 	private void updateCategories()
444 	{
445 		auto catfile = openFile("categories.json");
446 		scope(exit) catfile.close();
447 		auto json = parseJsonString(catfile.readAllUTF8());
448 
449 		Category[string] catmap;
450 
451 		Category processNode(Json node, string[] path)
452 		{
453 			path ~= node.name.get!string;
454 			auto cat = new Category;
455 			cat.name = path.join(".");
456 			cat.description = node.description.get!string;
457 			if (path.length > 2)
458 				cat.indentedDescription = "\u00a0\u00a0\u00a0\u00a0".replicate(path.length-2) ~ "\u00a0└ " ~ cat.description;
459 			else if (path.length == 2)
460 				cat.indentedDescription = "\u00a0└ " ~ cat.description;
461 			else cat.indentedDescription = cat.description;
462 			foreach_reverse (i; 0 .. path.length)
463 				if (existsFile("public/images/categories/"~path[0 .. i].join(".")~".png")) {
464 					cat.imageName = path[0 .. i].join(".");
465 					break;
466 				}
467 
468 			catmap[cat.name] = cat;
469 
470 			if ("categories" in node)
471 				foreach (subcat; node.categories)
472 					cat.subCategories ~= processNode(subcat, path);
473 
474 			return cat;
475 		}
476 
477 		Category[] cats;
478 		foreach (top_level_cat; json)
479 			cats ~= processNode(top_level_cat, null);
480 
481 		m_categories = cats;
482 		m_categoryMap = catmap;
483 	}
484 
485 	// Attribute for authenticated routes
486 	private enum auth = before!performAuth("_user");
487 	mixin PrivateAccessProxy;
488 
489 	private User performAuth(HTTPServerRequest req, HTTPServerResponse res)
490 	{
491 		return m_usermanauth.performAuth(req, res);
492 	}
493 }
494 
495 final class Category {
496 	string name, description, indentedDescription, imageName;
497 	Category[] subCategories;
498 }