1 /**
2 	A package supplier, able to get some packages to the local FS.
3 
4 	Copyright: © 2012-2013 Matthias Dondorff
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Matthias Dondorff
7 */
8 module dub.packagesupplier;
9 
10 import dub.dependency;
11 import dub.internal.utils;
12 import dub.internal.vibecompat.core.log;
13 import dub.internal.vibecompat.core.file;
14 import dub.internal.vibecompat.data.json;
15 import dub.internal.vibecompat.inet.url;
16 
17 import std.algorithm : filter, sort;
18 import std.array : array;
19 import std.conv;
20 import std.datetime;
21 import std.exception;
22 import std.file;
23 import std..string : format;
24 import std.zip;
25 
26 // TODO: drop the "best package" behavior and let retrievePackage/getPackageDescription take a Version instead of Dependency
27 
28 /// Supplies packages, this is done by supplying the latest possible version
29 /// which is available.
30 interface PackageSupplier {
31 	/// Returns a hunman readable representation of the supplier
32 	@property string description();
33 
34 	Version[] getVersions(string package_id);
35 
36 	/// path: absolute path to store the package (usually in a zip format)
37 	void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release);
38 
39 	/// returns the metadata for the package
40 	Json getPackageDescription(string packageId, Dependency dep, bool pre_release);
41 
42 	/// perform cache operation
43 	void cacheOp(Path cacheDir, CacheOp op);
44 }
45 
46 /// operations on package supplier cache
47 enum CacheOp {
48 	load,
49 	store,
50 	clean,
51 }
52 
53 class FileSystemPackageSupplier : PackageSupplier {
54 	private {
55 		Path m_path;
56 	}
57 
58 	this(Path root) { m_path = root; }
59 
60 	override @property string description() { return "file repository at "~m_path.toNativeString(); }
61 
62 	Version[] getVersions(string package_id)
63 	{
64 		Version[] ret;
65 		foreach (DirEntry d; dirEntries(m_path.toNativeString(), package_id~"*", SpanMode.shallow)) {
66 			Path p = Path(d.name);
67 			logDebug("Entry: %s", p);
68 			enforce(to!string(p.head)[$-4..$] == ".zip");
69 			auto vers = p.head.toString()[package_id.length+1..$-4];
70 			logDebug("Version: %s", vers);
71 			ret ~= Version(vers);
72 		}
73 		ret.sort();
74 		return ret;
75 	}
76 
77 	void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release)
78 	{
79 		enforce(path.absolute);
80 		logInfo("Storing package '"~packageId~"', version requirements: %s", dep);
81 		auto filename = bestPackageFile(packageId, dep, pre_release);
82 		enforce(existsFile(filename));
83 		copyFile(filename, path);
84 	}
85 
86 	Json getPackageDescription(string packageId, Dependency dep, bool pre_release)
87 	{
88 		auto filename = bestPackageFile(packageId, dep, pre_release);
89 		return jsonFromZip(filename, "dub.json");
90 	}
91 
92 	void cacheOp(Path cacheDir, CacheOp op) {
93 	}
94 
95 	private Path bestPackageFile(string packageId, Dependency dep, bool pre_release)
96 	{
97 		Path toPath(Version ver) {
98 			return m_path ~ (packageId ~ "-" ~ ver.toString() ~ ".zip");
99 		}
100 		auto versions = getVersions(packageId).filter!(v => dep.matches(v)).array;
101 		enforce(versions.length > 0, format("No package %s found matching %s", packageId, dep));
102 		foreach_reverse (ver; versions) {
103 			if (pre_release || !ver.isPreRelease)
104 				return toPath(ver);
105 		}
106 		return toPath(versions[$-1]);
107 	}
108 }
109 
110 
111 /// Client PackageSupplier using the registry available via registerVpmRegistry
112 class RegistryPackageSupplier : PackageSupplier {
113 	private {
114 		URL m_registryUrl;
115 		struct CacheEntry { Json data; SysTime cacheTime; }
116 		CacheEntry[string] m_metadataCache;
117 		Duration m_maxCacheTime;
118 		bool m_metadataCacheDirty;
119 	}
120 
121 	this(URL registry)
122 	{
123 		m_registryUrl = registry;
124 		m_maxCacheTime = 24.hours();
125 	}
126 
127 	override @property string description() { return "registry at "~m_registryUrl.toString(); }
128 
129 	Version[] getVersions(string package_id)
130 	{
131 		Version[] ret;
132 		Json md = getMetadata(package_id);
133 		foreach (json; md["versions"]) {
134 			auto cur = Version(cast(string)json["version"]);
135 			ret ~= cur;
136 		}
137 		ret.sort();
138 		return ret;
139 	}
140 
141 	void retrievePackage(Path path, string packageId, Dependency dep, bool pre_release)
142 	{
143 		import std.array : replace;
144 		Json best = getBestPackage(packageId, dep, pre_release);
145 		auto vers = best["version"].get!string;
146 		auto url = m_registryUrl ~ Path(PackagesPath~"/"~packageId~"/"~vers~".zip");
147 		logDiagnostic("Found download URL: '%s'", url);
148 		download(url, path);
149 	}
150 
151 	Json getPackageDescription(string packageId, Dependency dep, bool pre_release)
152 	{
153 		return getBestPackage(packageId, dep, pre_release);
154 	}
155 
156 	void cacheOp(Path cacheDir, CacheOp op)
157 	{
158 		auto path = cacheDir ~ cacheFileName;
159 		final switch (op)
160 		{
161 		case CacheOp.store:
162 			if (!m_metadataCacheDirty) return;
163 			if (!cacheDir.existsFile())
164 				mkdirRecurse(cacheDir.toNativeString());
165 			// TODO: method is slow due to Json escaping
166 			writeJsonFile(path, m_metadataCache.serializeToJson());
167 			break;
168 
169 		case CacheOp.load:
170 			if (!path.existsFile()) return;
171 			try deserializeJson(m_metadataCache, jsonFromFile(path));
172 			catch (Exception e) {
173 				import std.encoding;
174 				logWarn("Error loading package cache file %s: %s", path.toNativeString(), e.msg);
175 				logDebug("Full error: %s", e.toString().sanitize());
176 			}
177 			break;
178 
179 		case CacheOp.clean:
180 			if (path.existsFile()) removeFile(path);
181 			m_metadataCache.destroy();
182 			break;
183 		}
184 		m_metadataCacheDirty = false;
185 	}
186 
187 	private @property string cacheFileName()
188 	{
189 		import std.digest.md;
190 		auto hash = m_registryUrl.toString.md5Of();
191 		return m_registryUrl.host ~ hash[0 .. $/2].toHexString().idup ~ ".json";
192 	}
193 
194 	private Json getMetadata(string packageId)
195 	{
196 		auto now = Clock.currTime(UTC());
197 		if (auto pentry = packageId in m_metadataCache) {
198 			if (pentry.cacheTime + m_maxCacheTime > now)
199 				return pentry.data;
200 			m_metadataCache.remove(packageId);
201 			m_metadataCacheDirty = true;
202 		}
203 
204 		auto url = m_registryUrl ~ Path(PackagesPath ~ "/" ~ packageId ~ ".json");
205 
206 		logDebug("Downloading metadata for %s", packageId);
207 		logDebug("Getting from %s", url);
208 
209 		auto jsonData = cast(string)download(url);
210 		Json json = parseJsonString(jsonData, url.toString());
211 		// strip readme data (to save size and time)
212 		foreach (ref v; json["versions"])
213 			v.remove("readme");
214 		m_metadataCache[packageId] = CacheEntry(json, now);
215 		m_metadataCacheDirty = true;
216 		return json;
217 	}
218 
219 	private Json getBestPackage(string packageId, Dependency dep, bool pre_release)
220 	{
221 		Json md = getMetadata(packageId);
222 		Json best = null;
223 		Version bestver;
224 		foreach (json; md["versions"]) {
225 			auto cur = Version(cast(string)json["version"]);
226 			if (!dep.matches(cur)) continue;
227 			if (best == null) best = json;
228 			else if (pre_release) {
229 				if (cur > bestver) best = json;
230 			} else if (bestver.isPreRelease) {
231 				if (!cur.isPreRelease || cur > bestver) best = json;
232 			} else if (!cur.isPreRelease && cur > bestver) best = json;
233 			bestver = Version(cast(string)best["version"]);
234 		}
235 		enforce(best != null, "No package candidate found for "~packageId~" "~dep.toString());
236 		return best;
237 	}
238 }
239 
240 private enum PackagesPath = "packages";