1 /**
2 	Copyright: © 2013 rejectedsoftware e.K.
3 	License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file.
4 	Authors: Sönke Ludwig
5 */
6 module dubregistry.repositories.github;
7 
8 import dubregistry.cache;
9 import dubregistry.dbcontroller : DbRepository;
10 import dubregistry.repositories.repository;
11 import std.string : startsWith;
12 import std.typecons;
13 import vibe.core.log;
14 import vibe.core.stream;
15 import vibe.data.json;
16 import vibe.http.client : HTTPClientRequest;
17 import vibe.inet.url;
18 
19 
20 class GithubRepository : Repository {
21 @safe:
22 	private {
23 		string m_owner;
24 		string m_project;
25 		string m_authToken;
26 	}
27 
28 	static void register(string token)
29 	{
30 		Repository factory(DbRepository info) @safe {
31 			return new GithubRepository(info.owner, info.project, token);
32 		}
33 		addRepositoryFactory("github", &factory);
34 	}
35 
36 	this(string owner, string project, string auth_token)
37 	{
38 		m_owner = owner;
39 		m_project = project;
40 		m_authToken = auth_token;
41 	}
42 
43 	RefInfo[] getTags()
44 	{
45 		import std.datetime.systime : SysTime;
46 		import std.conv: text;
47 		RefInfo[] ret;
48 		Json[] tags;
49 		try tags = readPagedListFromRepo("/tags?per_page=100");
50 		catch( Exception e ) { throw new Exception("Failed to get tags: "~e.msg); }
51 		foreach_reverse (tag; tags) {
52 			try {
53 				auto tagname = tag["name"].get!string;
54 				Json commit = readJsonFromRepo("/commits/"~tag["commit"]["sha"].get!string, true, true);
55 				ret ~= RefInfo(tagname, tag["commit"]["sha"].get!string, SysTime.fromISOExtString(commit["commit"]["committer"]["date"].get!string));
56 				logDebug("Found tag for %s/%s: %s", m_owner, m_project, tagname);
57 			} catch( Exception e ){
58 				throw new Exception("Failed to process tag "~tag["name"].get!string~": "~e.msg);
59 			}
60 		}
61 		return ret;
62 	}
63 
64 	RefInfo[] getBranches()
65 	{
66 		import std.datetime.systime : SysTime;
67 
68 		Json branches = readJsonFromRepo("/branches");
69 		RefInfo[] ret;
70 		foreach_reverse( branch; branches ){
71 			auto branchname = branch["name"].get!string;
72 			Json commit = readJsonFromRepo("/commits/"~branch["commit"]["sha"].get!string, true, true);
73 			ret ~= RefInfo(branchname, branch["commit"]["sha"].get!string, SysTime.fromISOExtString(commit["commit"]["committer"]["date"].get!string));
74 			logDebug("Found branch for %s/%s: %s", m_owner, m_project, branchname);
75 		}
76 		return ret;
77 	}
78 
79 	RepositoryInfo getInfo()
80 	{
81 		auto nfo = readJsonFromRepo("");
82 		RepositoryInfo ret;
83 		ret.isFork = nfo["fork"].opt!bool;
84 		ret.stats.stars = nfo["stargazers_count"].opt!uint;
85 		ret.stats.watchers = nfo["subscribers_count"].opt!uint;
86 		ret.stats.forks = nfo["forks_count"].opt!uint;
87 		ret.stats.issues = nfo["open_issues_count"].opt!uint; // conflates PRs and Issues
88 		return ret;
89 	}
90 
91 	RepositoryFile[] listFiles(string commit_sha, InetPath path)
92 	{
93 		assert(path.absolute, "Passed relative path to listFiles.");
94 		auto url = "/contents"~path.toString()~"?ref="~commit_sha;
95 		auto ls = readJsonFromRepo(url).get!(Json[]);
96 		RepositoryFile[] ret;
97 		ret.reserve(ls.length);
98 		foreach (entry; ls) {
99 			string type = entry["type"].get!string;
100 			RepositoryFile file;
101 			if (type == "dir") {
102 				file.type = RepositoryFile.Type.directory;
103 			}
104 			else if (type == "file") {
105 				file.type = RepositoryFile.Type.file;
106 				file.size = entry["size"].get!size_t;
107 			}
108 			else continue;
109 			file.commitSha = commit_sha;
110 			file.path = InetPath("/" ~ entry["path"].get!string);
111 			ret ~= file;
112 		}
113 		return ret;
114 	}
115 
116 	void readFile(string commit_sha, InetPath path, scope void delegate(scope InputStream) @safe reader)
117 	{
118 		assert(path.absolute, "Passed relative path to readFile.");
119 		auto url = getContentURLPrefix()~"/"~m_owner~"/"~m_project~"/"~commit_sha~path.toString();
120 		downloadCached(url, (scope input) {
121 			reader(input);
122 		}, true, &addAuthentication);
123 	}
124 
125 	string getDownloadUrl(string ver)
126 	{
127 		import std.uri : encodeComponent;
128 		if( ver.startsWith("~") ) ver = ver[1 .. $];
129 		else ver = ver;
130 		auto venc = () @trusted { return encodeComponent(ver); } ();
131 		return "https://github.com/"~m_owner~"/"~m_project~"/archive/"~venc~".zip";
132 	}
133 
134 	void download(string ver, scope void delegate(scope InputStream) @safe del)
135 	{
136 		downloadCached(getDownloadUrl(ver), del, false, &addAuthentication);
137 	}
138 
139 	private Json readJsonFromRepo(string api_path, bool sanitize = false, bool cache_priority = false)
140 	{
141 		return readJson(getAPIURLPrefix()~"/repos/"~m_owner~"/"~m_project~api_path,
142 			sanitize, cache_priority, &addAuthentication);
143 	}
144 
145 	private Json[] readPagedListFromRepo(string api_path, bool sanitize = false, bool cache_priority = false)
146 	{
147 		return readPagedList(getAPIURLPrefix()~"/repos/"~m_owner~"/"~m_project~api_path,
148 			sanitize, cache_priority, &addAuthentication);
149 	}
150 
151 	private void addAuthentication(scope HTTPClientRequest req)
152 	{
153 		req.headers["Authorization"] = "token " ~ m_authToken;
154 	}
155 
156 	private string getAPIURLPrefix()
157 	{
158 		return "https://api.github.com";
159 	}
160 
161 	private string getContentURLPrefix()
162 	{
163 		return "https://raw.githubusercontent.com";
164 	}
165 }
166 
167 package Json[] readPagedList(string url, bool sanitize = false, bool cache_priority = false, RequestModifier request_modifier = null)
168 @safe {
169 	import dubregistry.internal.utils : black;
170 	import std.array : appender;
171 	import std.format : format;
172 	import vibe.stream.operations : readAllUTF8;
173 
174 	auto ret = appender!(Json[]);
175 	Exception ex;
176 	string next = url;
177 
178 	NextLoop: while (next.length) {
179 		logDiagnostic("Getting paged JSON response from %s", next.black);
180 		foreach (i; 0 .. 2) {
181 			try {
182 				downloadCached(next, (scope input, scope headers) {
183 					scope (failure) clearCacheEntry(url);
184 					next = getNextLink(headers);
185 
186 					auto text = input.readAllUTF8(sanitize);
187 					ret ~= parseJsonString(text).get!(Json[]);
188 				}, ["Link"], cache_priority, request_modifier);
189 				continue NextLoop;
190 			} catch (FileNotFoundException e) {
191 				throw e;
192 			} catch (Exception e) {
193 				logDiagnostic("Failed to parse downloaded JSON document (attempt #%s): %s", i+1, e.msg);
194 				ex = e;
195 			}
196 		}
197 		throw new Exception(format("Failed to read JSON from %s: %s", url.black, ex.msg), __FILE__, __LINE__, ex);
198 	}
199 
200 	return ret.data;
201 }
202 
203 private string getNextLink(scope string[string] headers)
204 @safe {
205 	import uritemplate : expandTemplateURIString;
206 	import std.algorithm : endsWith, splitter, startsWith;
207 
208 	static immutable string startPart = `<`;
209 	static immutable string endPart = `>; rel="next"`;
210 
211 	if (auto link = "Link" in headers) {
212 		foreach (part; (*link).splitter(", ")) {
213 			if (part.startsWith(startPart) && part.endsWith(endPart)) {
214 				return expandTemplateURIString(part[startPart.length .. $ - endPart.length], null);
215 			}
216 		}
217 	}
218 	return null;
219 }