1 /** 2 Copyright: © 2013 rejectedsoftware e.K. 3 License: Subject to the terms of the GNU GPLv3 license, as written in the included LICENSE.txt file. 4 Authors: Sönke Ludwig 5 */ 6 module dubregistry.repositories.github; 7 8 import dubregistry.cache; 9 import dubregistry.dbcontroller : DbRepository; 10 import dubregistry.repositories.repository; 11 import std.string : startsWith; 12 import std.typecons; 13 import vibe.core.log; 14 import vibe.core.stream; 15 import vibe.data.json; 16 import vibe.http.client : HTTPClientRequest; 17 import vibe.inet.url; 18 19 20 class GithubRepository : Repository { 21 @safe: 22 private { 23 string m_owner; 24 string m_project; 25 string m_authToken; 26 } 27 28 static void register(string token) 29 { 30 Repository factory(DbRepository info) @safe { 31 return new GithubRepository(info.owner, info.project, token); 32 } 33 addRepositoryFactory("github", &factory); 34 } 35 36 this(string owner, string project, string auth_token) 37 { 38 m_owner = owner; 39 m_project = project; 40 m_authToken = auth_token; 41 } 42 43 RefInfo[] getTags() 44 { 45 import std.datetime.systime : SysTime; 46 import std.conv: text; 47 RefInfo[] ret; 48 Json[] tags; 49 try tags = readPagedListFromRepo("/tags?per_page=100"); 50 catch( Exception e ) { throw new Exception("Failed to get tags: "~e.msg); } 51 foreach_reverse (tag; tags) { 52 try { 53 auto tagname = tag["name"].get!string; 54 Json commit = readJsonFromRepo("/commits/"~tag["commit"]["sha"].get!string, true, true); 55 ret ~= RefInfo(tagname, tag["commit"]["sha"].get!string, SysTime.fromISOExtString(commit["commit"]["committer"]["date"].get!string)); 56 logDebug("Found tag for %s/%s: %s", m_owner, m_project, tagname); 57 } catch( Exception e ){ 58 throw new Exception("Failed to process tag "~tag["name"].get!string~": "~e.msg); 59 } 60 } 61 return ret; 62 } 63 64 RefInfo[] getBranches() 65 { 66 import std.datetime.systime : SysTime; 67 68 Json branches = readJsonFromRepo("/branches"); 69 RefInfo[] ret; 70 foreach_reverse( branch; branches ){ 71 auto branchname = branch["name"].get!string; 72 Json commit = readJsonFromRepo("/commits/"~branch["commit"]["sha"].get!string, true, true); 73 ret ~= RefInfo(branchname, branch["commit"]["sha"].get!string, SysTime.fromISOExtString(commit["commit"]["committer"]["date"].get!string)); 74 logDebug("Found branch for %s/%s: %s", m_owner, m_project, branchname); 75 } 76 return ret; 77 } 78 79 RepositoryInfo getInfo() 80 { 81 auto nfo = readJsonFromRepo(""); 82 RepositoryInfo ret; 83 ret.isFork = nfo["fork"].opt!bool; 84 ret.stats.stars = nfo["stargazers_count"].opt!uint; 85 ret.stats.watchers = nfo["subscribers_count"].opt!uint; 86 ret.stats.forks = nfo["forks_count"].opt!uint; 87 ret.stats.issues = nfo["open_issues_count"].opt!uint; // conflates PRs and Issues 88 return ret; 89 } 90 91 RepositoryFile[] listFiles(string commit_sha, InetPath path) 92 { 93 assert(path.absolute, "Passed relative path to listFiles."); 94 auto url = "/contents"~path.toString()~"?ref="~commit_sha; 95 auto ls = readJsonFromRepo(url).get!(Json[]); 96 RepositoryFile[] ret; 97 ret.reserve(ls.length); 98 foreach (entry; ls) { 99 string type = entry["type"].get!string; 100 RepositoryFile file; 101 if (type == "dir") { 102 file.type = RepositoryFile.Type.directory; 103 } 104 else if (type == "file") { 105 file.type = RepositoryFile.Type.file; 106 file.size = entry["size"].get!size_t; 107 } 108 else continue; 109 file.commitSha = commit_sha; 110 file.path = InetPath("/" ~ entry["path"].get!string); 111 ret ~= file; 112 } 113 return ret; 114 } 115 116 void readFile(string commit_sha, InetPath path, scope void delegate(scope InputStream) @safe reader) 117 { 118 assert(path.absolute, "Passed relative path to readFile."); 119 auto url = getContentURLPrefix()~"/"~m_owner~"/"~m_project~"/"~commit_sha~path.toString(); 120 downloadCached(url, (scope input) { 121 reader(input); 122 }, true, &addAuthentication); 123 } 124 125 string getDownloadUrl(string ver) 126 { 127 import std.uri : encodeComponent; 128 if( ver.startsWith("~") ) ver = ver[1 .. $]; 129 else ver = ver; 130 auto venc = () @trusted { return encodeComponent(ver); } (); 131 return "https://github.com/"~m_owner~"/"~m_project~"/archive/"~venc~".zip"; 132 } 133 134 void download(string ver, scope void delegate(scope InputStream) @safe del) 135 { 136 downloadCached(getDownloadUrl(ver), del, false, &addAuthentication); 137 } 138 139 private Json readJsonFromRepo(string api_path, bool sanitize = false, bool cache_priority = false) 140 { 141 return readJson(getAPIURLPrefix()~"/repos/"~m_owner~"/"~m_project~api_path, 142 sanitize, cache_priority, &addAuthentication); 143 } 144 145 private Json[] readPagedListFromRepo(string api_path, bool sanitize = false, bool cache_priority = false) 146 { 147 return readPagedList(getAPIURLPrefix()~"/repos/"~m_owner~"/"~m_project~api_path, 148 sanitize, cache_priority, &addAuthentication); 149 } 150 151 private void addAuthentication(scope HTTPClientRequest req) 152 { 153 req.headers["Authorization"] = "token " ~ m_authToken; 154 } 155 156 private string getAPIURLPrefix() 157 { 158 return "https://api.github.com"; 159 } 160 161 private string getContentURLPrefix() 162 { 163 return "https://raw.githubusercontent.com"; 164 } 165 } 166 167 package Json[] readPagedList(string url, bool sanitize = false, bool cache_priority = false, RequestModifier request_modifier = null) 168 @safe { 169 import dubregistry.internal.utils : black; 170 import std.array : appender; 171 import std.format : format; 172 import vibe.stream.operations : readAllUTF8; 173 174 auto ret = appender!(Json[]); 175 Exception ex; 176 string next = url; 177 178 NextLoop: while (next.length) { 179 logDiagnostic("Getting paged JSON response from %s", next.black); 180 foreach (i; 0 .. 2) { 181 try { 182 downloadCached(next, (scope input, scope headers) { 183 scope (failure) clearCacheEntry(url); 184 next = getNextLink(headers); 185 186 auto text = input.readAllUTF8(sanitize); 187 ret ~= parseJsonString(text).get!(Json[]); 188 }, ["Link"], cache_priority, request_modifier); 189 continue NextLoop; 190 } catch (FileNotFoundException e) { 191 throw e; 192 } catch (Exception e) { 193 logDiagnostic("Failed to parse downloaded JSON document (attempt #%s): %s", i+1, e.msg); 194 ex = e; 195 } 196 } 197 throw new Exception(format("Failed to read JSON from %s: %s", url.black, ex.msg), __FILE__, __LINE__, ex); 198 } 199 200 return ret.data; 201 } 202 203 private string getNextLink(scope string[string] headers) 204 @safe { 205 import uritemplate : expandTemplateURIString; 206 import std.algorithm : endsWith, splitter, startsWith; 207 208 static immutable string startPart = `<`; 209 static immutable string endPart = `>; rel="next"`; 210 211 if (auto link = "Link" in headers) { 212 foreach (part; (*link).splitter(", ")) { 213 if (part.startsWith(startPart) && part.endsWith(endPart)) { 214 return expandTemplateURIString(part[startPart.length .. $ - endPart.length], null); 215 } 216 } 217 } 218 return null; 219 }