1 module dubregistry.internal.utils; 2 3 import vibe.core.core; 4 import vibe.core.concurrency; 5 import vibe.core.file; 6 import vibe.core.log; 7 import vibe.core.task; 8 import vibe.data.bson; 9 import vibe.inet.url; 10 import vibe.inet.path; 11 12 import core.time; 13 import std.algorithm : any, among, splitter; 14 import std.file : tempDir; 15 import std.format; 16 import std.path; 17 import std.process; 18 import std.string : indexOf, startsWith; 19 import std.typecons; 20 21 URL black(URL url) 22 @safe { 23 if (url.username.length > 0) url.username = "***"; 24 if (url.password.length > 0) url.password = "***"; 25 if (url.queryString.length > 0) { 26 size_t i; 27 char[] replace; 28 foreach (part; url.queryString.splitter('&')) { 29 if (part.startsWith("secret", "private")) { 30 if (!replace) 31 replace = url.queryString.dup; 32 auto eq = replace.indexOf('=', i); 33 if (eq != -1 && eq + 1 < i + part.length) { // only replace value if possible (key=value) 34 // +1 to pass the '=' character 35 replace = replace[0 .. (eq + 1)] ~ "***" ~ replace[i + part.length .. $]; 36 i = eq + 4; 37 } else { // otherwise replace the whole pair 38 replace = replace[0 .. i] ~ "***" ~ replace[i + part.length .. $]; 39 i += 3; 40 } 41 } else { 42 i += part.length; 43 } 44 i++; // '&' 45 } 46 if (replace) 47 url.queryString = (() @trusted => cast(string) replace)(); 48 } 49 return url; 50 } 51 52 string black(string url) 53 @safe { 54 return black(URL(url)).toString(); 55 } 56 57 @safe unittest 58 { 59 assert(black("https://root:root@google.com/") == "https://***:***@google.com/"); 60 assert(black("https://root:root@google.com/?secret=12345") == "https://***:***@google.com/?secret=***"); 61 assert(black("https://root:root@google.com/?secret_12345") == "https://***:***@google.com/?***"); 62 assert(black("https://root:root@google.com/?secret_12345&private=2") == "https://***:***@google.com/?***&private=***"); 63 assert(black("https://root:root@google.com/?secret_12345&private=2&") == "https://***:***@google.com/?***&private=***&"); 64 assert(black("https://root:root@google.com/?secret_12345&private=2&a=b") == "https://***:***@google.com/?***&private=***&a=b"); 65 } 66 67 /** 68 * Params: 69 * file = the file to convert 70 * Returns: the PNG stream of the icon or empty on failure 71 * Throws: Exception if input is invalid format, invalid dimension or times out 72 */ 73 bdata_t generateLogo(NativePath file) @trusted 74 { 75 import std.concurrency : send, receiveOnly, Tid; 76 static assert (isWeaklyIsolated!(typeof(&generateLogoUnsafe))); 77 static assert (isWeaklyIsolated!NativePath); 78 static assert (isWeaklyIsolated!LogoGenerateResponse); 79 80 runWorkerTask((NativePath file, Tid par) { par.send(generateLogoUnsafe(file)); }, file, thisTid); 81 auto res = receiveOnly!LogoGenerateResponse(); 82 if (res.error.length) 83 throw new Exception("Failed to generate logo: " ~ res.error); 84 return res.data; 85 } 86 87 private struct LogoGenerateResponse 88 { 89 bdata_t data; 90 string error; 91 } 92 93 // need to return * here because stack returned values get destroyed for some reason... 94 private LogoGenerateResponse generateLogoUnsafe(NativePath file) @safe nothrow 95 { 96 import std.array : appender; 97 98 // TODO: replace imagemagick command line tools with something like imageformats on dub 99 100 // use [0] to only get first frame in gifs, has no effect on static images. 101 string firstFrame = file.toNativeString ~ "[0]"; 102 103 try { 104 auto sizeInfo = execute(["identify", "-format", "%w %h %m", firstFrame]); 105 if (sizeInfo.status != 0) 106 return LogoGenerateResponse(null, "Malformed image."); 107 int width, height; 108 string format; 109 uint filled = formattedRead(sizeInfo.output, "%d %d %s", width, height, format); 110 if (filled < 3) 111 return LogoGenerateResponse(null, "Malformed metadata."); 112 if (!format.among("PNG", "JPEG", "GIF", "BMP")) 113 return LogoGenerateResponse(null, "Invalid image format, only supporting png, jpeg, gif and bmp."); 114 if (width < 2 || height < 2 || width > 2048 || height > 2048) 115 return LogoGenerateResponse(null, "Invalid image dimenstions, must be between 2x2 and 2048x2048."); 116 117 auto png = pipeProcess(["timeout", "3", "convert", firstFrame, "-resize", "512x512>", "png:-"]); 118 119 auto a = appender!(immutable(ubyte)[])(); 120 121 (() @trusted { 122 foreach (chunk; png.stdout.byChunk(4096)) 123 a.put(chunk); 124 })(); 125 126 auto result = png.pid.wait; 127 if (result != 0) 128 { 129 if (result == 126) 130 return LogoGenerateResponse(null, "Conversion timed out"); 131 (() @trusted { 132 foreach (error; png.stderr.byLine) 133 logDiagnostic("convert error: %s", error); 134 })(); 135 return LogoGenerateResponse(null, "An unexpected error occured"); 136 } 137 138 return LogoGenerateResponse(a.data); 139 } catch (Exception e) { 140 return LogoGenerateResponse(null, "Failed to invoke the logo conversion process."); 141 } 142 } 143 144 145 /** Performs deduplication and compact re-allocation of individual strings. 146 147 Strings are allocated out of 64KB blocks of memory. 148 */ 149 struct PackedStringAllocator { 150 private { 151 char[] memory; 152 string[string] map; 153 } 154 155 @disable this(this); 156 157 string alloc(in char[] chars) 158 @safe { 159 import std.algorithm.comparison : max; 160 161 if (auto pr = chars in map) 162 return *pr; 163 164 if (memory.length < chars.length) memory = new char[](max(chars.length, 64*1024)); 165 auto str = memory[0 .. chars.length]; 166 memory = memory[chars.length .. $]; 167 str[] = chars[]; 168 auto istr = () @trusted { return cast(string)str; } (); 169 map[istr] = istr; 170 return istr; 171 } 172 }