1 /** 2 A package supplier, able to get some packages to the local FS. 3 4 Copyright: © 2012-2013 Matthias Dondorff 5 License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file. 6 Authors: Matthias Dondorff 7 */ 8 module dub.packagesupplier; 9 10 import dub.dependency; 11 import dub.internal.utils; 12 import dub.internal.vibecompat.core.log; 13 import dub.internal.vibecompat.core.file; 14 import dub.internal.vibecompat.data.json; 15 import dub.internal.vibecompat.inet.url; 16 17 import std.algorithm : filter, sort; 18 import std.array : array; 19 import std.conv; 20 import std.datetime; 21 import std.exception; 22 import std.file; 23 import std..string : format; 24 import std.zip; 25 26 // TODO: drop the "best package" behavior and let retrievePackage/getPackageDescription take a Version instead of Dependency 27 28 /// Supplies packages, this is done by supplying the latest possible version 29 /// which is available. 30 interface PackageSupplier { 31 /// Returns a hunman readable representation of the supplier 32 @property string description(); 33 34 Version[] getVersions(string package_id); 35 36 /// path: absolute path to store the package (usually in a zip format) 37 void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release); 38 39 /// returns the metadata for the package 40 Json getPackageDescription(string packageId, Dependency dep, bool pre_release); 41 42 /// perform cache operation 43 void cacheOp(Path cacheDir, CacheOp op); 44 } 45 46 /// operations on package supplier cache 47 enum CacheOp { 48 load, 49 store, 50 clean, 51 } 52 53 class FileSystemPackageSupplier : PackageSupplier { 54 private { 55 Path m_path; 56 } 57 58 this(Path root) { m_path = root; } 59 60 override @property string description() { return "file repository at "~m_path.toNativeString(); } 61 62 Version[] getVersions(string package_id) 63 { 64 Version[] ret; 65 foreach (DirEntry d; dirEntries(m_path.toNativeString(), package_id~"*", SpanMode.shallow)) { 66 Path p = Path(d.name); 67 logDebug("Entry: %s", p); 68 enforce(to!string(p.head)[$-4..$] == ".zip"); 69 auto vers = p.head.toString()[package_id.length+1..$-4]; 70 logDebug("Version: %s", vers); 71 ret ~= Version(vers); 72 } 73 ret.sort(); 74 return ret; 75 } 76 77 void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release) 78 { 79 enforce(path.absolute); 80 logInfo("Storing package '"~packageId~"', version requirements: %s", dep); 81 auto filename = bestPackageFile(packageId, dep, pre_release); 82 enforce(existsFile(filename)); 83 copyFile(filename, path); 84 } 85 86 Json getPackageDescription(string packageId, Dependency dep, bool pre_release) 87 { 88 auto filename = bestPackageFile(packageId, dep, pre_release); 89 return jsonFromZip(filename, "dub.json"); 90 } 91 92 void cacheOp(Path cacheDir, CacheOp op) { 93 } 94 95 private Path bestPackageFile(string packageId, Dependency dep, bool pre_release) 96 { 97 Path toPath(Version ver) { 98 return m_path ~ (packageId ~ "-" ~ ver.toString() ~ ".zip"); 99 } 100 auto versions = getVersions(packageId).filter!(v => dep.matches(v)).array; 101 enforce(versions.length > 0, format("No package %s found matching %s", packageId, dep)); 102 foreach_reverse (ver; versions) { 103 if (pre_release || !ver.isPreRelease) 104 return toPath(ver); 105 } 106 return toPath(versions[$-1]); 107 } 108 } 109 110 111 /// Client PackageSupplier using the registry available via registerVpmRegistry 112 class RegistryPackageSupplier : PackageSupplier { 113 private { 114 URL m_registryUrl; 115 struct CacheEntry { Json data; SysTime cacheTime; } 116 CacheEntry[string] m_metadataCache; 117 Duration m_maxCacheTime; 118 bool m_metadataCacheDirty; 119 } 120 121 this(URL registry) 122 { 123 m_registryUrl = registry; 124 m_maxCacheTime = 24.hours(); 125 } 126 127 override @property string description() { return "registry at "~m_registryUrl.toString(); } 128 129 Version[] getVersions(string package_id) 130 { 131 Version[] ret; 132 Json md = getMetadata(package_id); 133 foreach (json; md["versions"]) { 134 auto cur = Version(cast(string)json["version"]); 135 ret ~= cur; 136 } 137 ret.sort(); 138 return ret; 139 } 140 141 void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release) 142 { 143 import std.array : replace; 144 Json best = getBestPackage(packageId, dep, pre_release); 145 auto vers = best["version"].get!string; 146 auto url = m_registryUrl ~ Path(PackagesPath~"/"~packageId~"/"~vers~".zip"); 147 logDiagnostic("Found download URL: '%s'", url); 148 download(url, path); 149 } 150 151 Json getPackageDescription(string packageId, Dependency dep, bool pre_release) 152 { 153 return getBestPackage(packageId, dep, pre_release); 154 } 155 156 void cacheOp(Path cacheDir, CacheOp op) 157 { 158 auto path = cacheDir ~ cacheFileName; 159 final switch (op) 160 { 161 case CacheOp.store: 162 if (!m_metadataCacheDirty) return; 163 if (!cacheDir.existsFile()) 164 mkdirRecurse(cacheDir.toNativeString()); 165 // TODO: method is slow due to Json escaping 166 writeJsonFile(path, m_metadataCache.serializeToJson()); 167 break; 168 169 case CacheOp.load: 170 if (!path.existsFile()) return; 171 try deserializeJson(m_metadataCache, jsonFromFile(path)); 172 catch (Exception e) { 173 import std.encoding; 174 logWarn("Error loading package cache file %s: %s", path.toNativeString(), e.msg); 175 logDebug("Full error: %s", e.toString().sanitize()); 176 } 177 break; 178 179 case CacheOp.clean: 180 if (path.existsFile()) removeFile(path); 181 m_metadataCache.destroy(); 182 break; 183 } 184 m_metadataCacheDirty = false; 185 } 186 187 private @property string cacheFileName() 188 { 189 import std.digest.md; 190 auto hash = m_registryUrl.toString.md5Of(); 191 return m_registryUrl.host ~ hash[0 .. $/2].toHexString().idup ~ ".json"; 192 } 193 194 private Json getMetadata(string packageId) 195 { 196 auto now = Clock.currTime(UTC()); 197 if (auto pentry = packageId in m_metadataCache) { 198 if (pentry.cacheTime + m_maxCacheTime > now) 199 return pentry.data; 200 m_metadataCache.remove(packageId); 201 m_metadataCacheDirty = true; 202 } 203 204 auto url = m_registryUrl ~ Path(PackagesPath ~ "/" ~ packageId ~ ".json"); 205 206 logDebug("Downloading metadata for %s", packageId); 207 logDebug("Getting from %s", url); 208 209 auto jsonData = cast(string)download(url); 210 Json json = parseJsonString(jsonData, url.toString()); 211 // strip readme data (to save size and time) 212 foreach (ref v; json["versions"]) 213 v.remove("readme"); 214 m_metadataCache[packageId] = CacheEntry(json, now); 215 m_metadataCacheDirty = true; 216 return json; 217 } 218 219 private Json getBestPackage(string packageId, Dependency dep, bool pre_release) 220 { 221 Json md = getMetadata(packageId); 222 Json best = null; 223 Version bestver; 224 foreach (json; md["versions"]) { 225 auto cur = Version(cast(string)json["version"]); 226 if (!dep.matches(cur)) continue; 227 if (best == null) best = json; 228 else if (pre_release) { 229 if (cur > bestver) best = json; 230 } else if (bestver.isPreRelease) { 231 if (!cur.isPreRelease || cur > bestver) best = json; 232 } else if (!cur.isPreRelease && cur > bestver) best = json; 233 bestver = Version(cast(string)best["version"]); 234 } 235 enforce(best != null, "No package candidate found for "~packageId~" "~dep.toString()); 236 return best; 237 } 238 } 239 240 private enum PackagesPath = "packages";