Implement worker based image download, configuration & flashing

README.md

@@ -3,7 +3,7 @@
 [![GitHub Issues](https://img.shields.io/github/issues/balena-io/leviathan.svg)](https://github.com/balena-io/leviathan/issues)
 [![GitHub pull requests](https://img.shields.io/github/issues-pr/balena-io/leviathan.svg)](https://github.com/balena-io/leviathan/pulls)
 [![node](https://img.shields.io/badge/node-v12.0.0-green.svg)](https://nodejs.org/download/release/v12.0.0/)
-[![License](https://img.shields.io/badge/license-APACHE%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)  [![balena deploy button](https://www.balena.io/deploy.svg)](https://dashboard.balena-cloud.com/deploy?repoUrl=https://github.com/balena-os/leviathan)
+[![License](https://img.shields.io/badge/license-APACHE%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 Leviathan can be used to automate, test and control real/virtual devices in production or controlled environments. What can you do:
 

client/lib/index.js

@@ -18,14 +18,6 @@ const { parse } = require('url');
 const WebSocket = require('ws');
 const zlib = require('zlib');
 
-async function isGzip(filePath) {
-	const buf = Buffer.alloc(3);
-
-	await fs.read(await fs.open(filePath, 'r'), buf, 0, 3, 0);
-
-	return buf[0] === 0x1f && buf[1] === 0x8b && buf[2] === 0x08;
-}
-
 async function getFilesFromDirectory(basePath, ignore = []) {
 	let files = [];
 
@@ -98,31 +90,6 @@ module.exports = class Client extends PassThrough {
 				if (!stat[artifact.type]()) {
 					throw new Error(`${artifact.path} does not satisfy ${artifact.type}`);
 				}
-
-				if (artifact.name === 'os.img' && !(await isGzip(artifact.path))) {
-					const str = progStream({
-						length: stat.size,
-						time: 100,
-					});
-					str.on('progress', (progress) => {
-						this.status({
-							message: 'Gzipping Image',
-							percentage: progress.percentage,
-							eta: progress.eta,
-						});
-					});
-
-					const gzippedPath = join(this.workdir, artifact.name);
-
-					await pipeline(
-						fs.createReadStream(artifact.path),
-						str,
-						zlib.createGzip({ level: 6 }),
-						fs.createWriteStream(gzippedPath),
-					);
-
-					artifact.path = gzippedPath;
-				}
 			} catch (err) {
 				console.log(err);
 			}
@@ -320,21 +287,6 @@ module.exports = class Client extends PassThrough {
 									artifact.path = makePath(suiteConfig.suite);
 									artifact.type = 'isDirectory';
 									break;
-								case 'image':
-									// [Hack] Upload a fake image if image is false
-									// Remove when https://github.com/balena-os/leviathan/issues/567 is resolved
-									if (suiteConfig.image === false) {
-										// Had to create fake image in home directory otherwise
-										// facing a permission issue since client root is read-only
-										const fakeImagePath = '/home/test-balena.img.gz';
-										await fs.writeFile(fakeImagePath, '');
-										artifact.path = makePath(fakeImagePath);
-										artifact.type = 'isFile';
-										break;
-									}
-									artifact.path = makePath(suiteConfig.image);
-									artifact.type = 'isFile';
-									break;
 								case 'config':
 									artifact.type = 'isJSON';
 									artifact.data = null;

client/lib/schemas/multi-client-config.js

@@ -18,7 +18,8 @@ const innerSchema = {
 			],
 		},
 		image: {
-			type: ['string', 'boolean'],
+			type: ['string'],
+			format: 'uri'
 		},
 		workers: {
 			oneOf: [

core/lib/common/worker.js

@@ -111,75 +111,14 @@ module.exports = class Worker {
 	/**
 	 * Flash the provided OS image onto the connected DUT
 	 *
-	 * @param {string} imagePath path of the image to be flashed onto the DUT
+	 * @param {string} imageUrl url of the image to be flashed onto the DUT
 	 *
 	 * @category helper
 	 */
-	async flash(imagePath) {
-		let attempt = 0;
-		await retry(
-			async () => {
-				attempt++;
-				this.logger.log(`Preparing to flash, attempt ${attempt}...`);
-
-				await new Promise(async (resolve, reject) => {
-					const req = rp.post({ uri: `${this.url}/dut/flash` });
-
-					req.catch((error) => {
-						reject(error);
-					});
-					req.finally(() => {
-						if (lastStatus !== 'done') {
-							reject(new Error('Unexpected end of TCP connection'));
-						}
-
-						resolve();
-					});
-
-					let lastStatus;
-					req.on('data', (data) => {
-						const computedLine = RegExp('(.+?): (.*)').exec(data.toString());
-
-						if (computedLine) {
-							if (computedLine[1] === 'error') {
-								req.cancel();
-								reject(new Error(computedLine[2]));
-							}
-
-							if (computedLine[1] === 'progress') {
-								once(() => {
-									this.logger.log('Flashing');
-								});
-								// Hide any errors as the lines we get can be half written
-								const state = JSON.parse(computedLine[2]);
-								if (state != null && isNumber(state.percentage)) {
-									this.logger.status({
-										message: 'Flashing',
-										percentage: state.percentage,
-									});
-								}
-							}
-
-							if (computedLine[1] === 'status') {
-								lastStatus = computedLine[2];
-							}
-						}
-					});
-
-					pipeline(
-						fs.createReadStream(imagePath),
-						createGzip({ level: 6 }),
-						req,
-					);
-				});
-				this.logger.log('Flash completed');
-			},
-			{
-				max_tries: 5,
-				interval: 1000 * 5,
-				throw_original: true,
-			},
-		);
+	async flash(imageUrl) {
+		// Do we even need to send the url of the image if the download is being done by the fetch function in balenaOS 
+		// Something to think about for sure. 
+		// The tests would need to change if we don't add any argument
 	}
 
 
@@ -490,24 +429,6 @@ module.exports = class Worker {
 		}
 	}
 
-	// sends file over rsync
-	async sendFile(filePath, destination, target) {
-		if (target === 'worker') {
-			let containerId = await this.executeCommandInWorkerHost(
-				`balena ps | grep worker | awk '{print $1}'`,
-			);
-			// todo : replace with npm package
-			await exec(
-				`rsync -av -e "ssh ${this.workerUser}@${this.workerHost} -p ${this.workerPort} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q ${this.sshPrefix}balena exec -i" ${filePath} ${containerId}:${destination}`,
-			);
-		} else {
-			let ip = await this.ip(target);
-			await exec(
-				`rsync -av -e "ssh -p 22222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -q -i ${this.sshKey}" ${filePath} root@${ip}:${destination}`,
-			);
-		}
-	}
-
 	// add ssh key to the worker, so it can ssh into prod DUT's
 	async addSSHKey(keyPath) {
 		if (!this.directConnect) {

core/lib/components/balena/cli.js

@@ -66,47 +66,7 @@ module.exports = class CLI {
 	 * @category helper
 	 */
 	async preload(image, options) {
-		this.logger.log('--Preloading image--');
-		this.logger.log(`Image path: ${image}`);
-		this.logger.log(`Fleet: ${options.app}`);
-		this.logger.log(`Commit: ${options.commit}`);
-		await new Promise((resolve, reject) => {
-			const output = [];
-			const child = spawn(
-				'balena',
-				[
-					`preload ${image} --fleet ${options.app} --commit ${
-						options.commit
-					} ${options.pin ? '--pin-device-to-release ' : ''}`,
-					'--debug'
-				],
-				{
-					stdio: 'inherit',
-					shell: true,
-				},
-			);
-
-			function handleSignal(signal) {
-				child.kill(signal);
-			}
-
-			process.on('SIGINT', handleSignal);
-			process.on('SIGTERM', handleSignal);
-			child.on('exit', (code) => {
-				process.off('SIGINT', handleSignal);
-				process.off('SIGTERM', handleSignal);
-				if (code === 0) {
-					resolve();
-				} else {
-					reject()
-				}
-			});
-			child.on('error', (err) => {
-				process.off('SIGINT', handleSignal);
-				process.off('SIGTERM', handleSignal);
-				reject(err);
-			});
-		});
+		// Trigger preload on the worker for the image it already has
 	}
 
 	/**
@@ -118,6 +78,9 @@ module.exports = class CLI {
 	 * @category helper
 	 */
 	push(target, options) {
+
+		//  Reconsider this, I forgot why we used to local push from core on unmanaged OS tests. 
+
 		this.logger.log('Performing local push');
 		return exec(
 			`balena push ${target} --source ${options.source} --nolive --detached`,

core/lib/components/balena/sdk.js

@@ -335,61 +335,7 @@ module.exports = class BalenaSDK {
 	 * @category helper
 	 */
 	async fetchOS(versionOrRange = 'latest', deviceType, osType = 'default') {
-		// normalize the version string/range, supports 'latest', 'recommended', etc
-		let version = await this.balena.models.os.getMaxSatisfyingVersion(
-			deviceType,
-			versionOrRange,
-			osType,
-		);
-
-		// variant is deprecated in recent balenaOS releases but
-		// if prod variant is still present after being normalized, replace it with dev
-		version = version.replace('.prod', '.dev');
-
-		const path = join(
-			config.leviathan.downloads,
-			`balenaOs-${version}.img`,
-		);
-
-		// Caching implementation if needed - Check https://github.com/balena-os/leviathan/issues/441
+		// 	Code to trigger download of production balenaOS on the worker 
 
-		let attempt = 0;
-		const downloadLatestOS = async () => {
-			attempt++;
-			this.logger.log(
-				`Fetching balenaOS version ${version}, attempt ${attempt}...`,
-			);
-			return await new Promise(async (resolve, reject) => {
-				await this.balena.models.os.download(
-					deviceType,
-					version,
-					function (error, stream) {
-						if (error) {
-							fs.unlink(path, () => {
-								// Ignore.
-							});
-							reject(`Image download failed: ${error}`);
-						}
-						// Shows progress of image download
-						let progress = 0;
-						stream.on('progress', (data) => {
-							if (data.percentage >= progress + 10) {
-								console.log(
-									`Downloading balenaOS image: ${toInteger(data.percentage) + '%'
-									}`,
-								);
-								progress = data.percentage;
-							}
-						});
-						stream.pipe(fs.createWriteStream(path));
-						stream.on('finish', () => {
-							console.log(`Download Successful: ${path}`);
-							resolve(path);
-						});
-					},
-				);
-			});
-		};
-		return retry(downloadLatestOS, { max_tries: 3, interval: 500 });
 	}
-};
+}