From 37f43709900034a21fb7c54b093c31687cd9a62f Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Mon, 25 Nov 2024 11:25:40 -0500 Subject: [PATCH 1/3] fix(weavedrive): remove tx2 data items --- extensions/weavedrive/client/main.lua | 63 +-- extensions/weavedrive/src/index.js | 466 ++++++++--------------- extensions/weavedrive/test/index.test.js | 118 ++---- 3 files changed, 213 insertions(+), 434 deletions(-) diff --git a/extensions/weavedrive/client/main.lua b/extensions/weavedrive/client/main.lua index 22ed2c8f..ca9dabdb 100644 --- a/extensions/weavedrive/client/main.lua +++ b/extensions/weavedrive/client/main.lua @@ -6,54 +6,33 @@ local drive = { _version = "0.0.1" } function drive.getBlock(height) - local block = io.open("/block/" .. height) - if not block then - return nil, "Block Header not found!" - end - local headers = require('json').decode( - block:read( - block:seek('end') - ) - ) - block:close() - return headers + local block = io.open("/block/" .. height) + if not block then + return nil, "Block Header not found!" + end + local headers = require("json").decode(block:read(block:seek("end"))) + block:close() + return headers end function drive.getTx(txId) - local file = io.open('/tx/' .. txId) - if not file then - return nil, "File not found!" - end - local contents = require('json').decode( - file:read( - file:seek('end') - ) - ) - file:close() - return contents + local file = io.open("/tx/" .. txId) + if not file then + return nil, "File not found!" + end + local contents = require("json").decode(file:read(file:seek("end"))) + file:close() + return contents end function drive.getData(txId) - local file = io.open('/data/' .. txId) - if not file then - return nil, "File not found!" - end - local contents = file:read( - file:seek('end') - ) - file:close() - return contents + local file = io.open("/data/" .. txId) + if not file then + return nil, "File not found!" + end + local contents = file:read(file:seek("end")) + file:close() + return contents end -function drive.getDataItem(txId) - local file = io.open('/tx2/' .. txId) - if not file then - return nil, "File not found!" - end - local contents = file:read( - file:seek('end') - ) - file:close() - return contents -end return drive diff --git a/extensions/weavedrive/src/index.js b/extensions/weavedrive/src/index.js index f6b73198..fbb12c38 100644 --- a/extensions/weavedrive/src/index.js +++ b/extensions/weavedrive/src/index.js @@ -1,6 +1,4 @@ -var assert = require('assert') -var Arweave = require('arweave') - +const Arweave = require('arweave') const KB = 1024 const MB = KB * 1024 @@ -8,10 +6,10 @@ const CACHE_SZ = 32 * KB const CHUNK_SZ = 128 * MB const NOTIFY_SZ = 512 * MB -module.exports = function weaveDrive(mod, FS) { +module.exports = function weaveDrive (mod, FS) { return { - reset(fd) { - //console.log("WeaveDrive: Resetting fd: ", fd) + reset (fd) { + // console.log("WeaveDrive: Resetting fd: ", fd) FS.streams[fd].node.position = 0 FS.streams[fd].node.cache = new Uint8Array(0) }, @@ -19,18 +17,18 @@ module.exports = function weaveDrive(mod, FS) { joinUrl ({ url, path }) { if (!path) return url if (path.startsWith('/')) return this.joinUrl({ url, path: path.slice(1) }) - + url = new URL(url) url.pathname += path return url.toString() }, - async customFetch(path, options) { + async customFetch (path, options) { /** * mod.ARWEAVE may be a comma-delimited list of urls. * So we parse it into an array that we sequentially consume * using fetch, and return the first successful response. - * + * * The first url is considered "primary". So if all urls fail * to produce a successful response, then we return the primary's * error response @@ -53,12 +51,12 @@ module.exports = function weaveDrive(mod, FS) { return p }, - async create(id) { - var properties = { isDevice: false, contents: null } + async create (id) { + const properties = { isDevice: false, contents: null } if (!await this.checkAdmissible(id)) { - //console.log("WeaveDrive: Arweave ID is not admissable! ", id) - return 0; + // console.log("WeaveDrive: Arweave ID is not admissable! ", id) + return 0 } // Create the file in the emscripten FS @@ -67,334 +65,229 @@ module.exports = function weaveDrive(mod, FS) { // called first because were only loading Data, we needed to create // the directory. See: https://github.com/permaweb/aos/issues/342 if (!FS.analyzePath('/data/').exists) { - FS.mkdir('/data/'); + FS.mkdir('/data/') } - var node = FS.createFile('/', 'data/' + id, properties, true, false); + const node = FS.createFile('/', 'data/' + id, properties, true, false) // Set initial parameters - var bytesLength = await this.customFetch(`/${id}`, { method: 'HEAD' }).then(res => res.headers.get('Content-Length')) + const bytesLength = await this.customFetch(`/${id}`, { method: 'HEAD' }).then(res => res.headers.get('Content-Length')) node.total_size = Number(bytesLength) node.cache = new Uint8Array(0) - node.position = 0; + node.position = 0 // Add a function that defers querying the file size until it is asked the first time. - Object.defineProperties(node, { usedBytes: { get: function () { return bytesLength; } } }); + Object.defineProperties(node, { usedBytes: { get: function () { return bytesLength } } }) // Now we have created the file in the emscripten FS, we can open it as a stream - var stream = FS.open('/data/' + id, "r") + const stream = FS.open('/data/' + id, 'r') - //console.log("JS: Created file: ", id, " fd: ", stream.fd); - return stream; + // console.log("JS: Created file: ", id, " fd: ", stream.fd); + return stream }, - async createBlockHeader(id) { + async createBlockHeader (id) { const customFetch = this.customFetch // todo: add a bunch of retries - async function retry(x) { - return new Promise(r => { + async function retry (x) { + return new Promise(resolve => { setTimeout(function () { - r(customFetch(`/block/height/${id}`)) + resolve(customFetch(`/block/height/${id}`)) }, x * 10000) }) } - var result = await this.customFetch(`/block/height/${id}`) + const result = await this.customFetch(`/block/height/${id}`) .then(res => !res.ok ? retry(1) : res) .then(res => !res.ok ? retry(2) : res) .then(res => !res.ok ? retry(3) : res) .then(res => !res.ok ? retry(4) : res) - .then(res => res.text()); - - var bytesLength = result.length; + .then(res => res.text()) - var node = FS.createDataFile('/', 'block/' + id, Buffer.from(result, 'utf-8'), true, false); + FS.createDataFile('/', 'block/' + id, Buffer.from(result, 'utf-8'), true, false) - var stream = FS.open('/block/' + id, 'r'); - return stream; + const stream = FS.open('/block/' + id, 'r') + return stream }, - async createTxHeader(id) { + async createTxHeader (id) { const customFetch = this.customFetch - async function toAddress(owner) { + async function toAddress (owner) { return Arweave.utils.bufferTob64Url( await Arweave.crypto.hash(Arweave.utils.b64UrlToBuffer(owner)) - ); + ) } - async function retry(x) { - return new Promise(r => { + async function retry (x) { + return new Promise(resolve => { setTimeout(function () { - r(customFetch(`/tx/${id}`)) + resolve(customFetch(`/tx/${id}`)) }, x * 10000) }) } // todo: add a bunch of retries - var result = await this.customFetch(`/tx/${id}`) + const result = await this.customFetch(`/tx/${id}`) .then(res => !res.ok ? retry(1) : res) .then(res => !res.ok ? retry(2) : res) .then(res => !res.ok ? retry(3) : res) .then(res => !res.ok ? retry(4) : res) .then(res => res.json()) .then(async entry => ({ ...entry, ownerAddress: await toAddress(entry.owner) })) - //.then(x => (console.error(x), x)) - .then(x => JSON.stringify(x)); - - var node = FS.createDataFile('/', 'tx/' + id, Buffer.from(result, 'utf-8'), true, false); - var stream = FS.open('/tx/' + id, 'r'); - return stream; - }, - async createDataItemTxHeader(id) { - const gqlQuery = this.gqlQuery - var GET_TRANSACTION_QUERY = ` - query GetTransactions ($transactionIds: [ID!]!) { - transactions(ids: $transactionIds) { - edges { - node { - id - anchor - data { - size - } - signature - recipient - owner { - address - key - } - fee { - ar - winston - } - quantity { - winston - ar - } - tags { - name - value - } - bundledIn { - id - } - block { - id - timestamp - height - previous - } - } - } - } - }` - var variables = { transactionIds: [id] } - async function retry(x) { - return new Promise(r => { - setTimeout(function () { - r(gqlQuery(GET_TRANSACTION_QUERY, variables)) - }, x * 10000) - }) - } - - const gqlExists = await this.gqlExists() - if (!gqlExists) { - return 'GQL Not Found!' - } - - // todo: add a bunch of retries - var result = await this.gqlQuery(GET_TRANSACTION_QUERY, variables) - .then(res => !res.ok ? retry(1) : res) - .then(res => !res.ok ? retry(2) : res) - .then(res => !res.ok ? retry(3) : res) - .then(res => !res.ok ? retry(4) : res) - .then(res => res.json()) - .then(res => { - return res?.data?.transactions?.edges?.[0]?.node ? res.data.transactions.edges[0].node : 'No results' - }) - .then(async entry => { - return typeof (entry) == 'string' ? entry : { - format: 3, - ...entry - } - }) - .then(x => { - return typeof (x) == 'string' ? x : JSON.stringify(x) - }); - - - if (result === 'No results') { - return result - } - FS.createDataFile('/', 'tx2/' + id, Buffer.from(result, 'utf-8'), true, false); - var stream = FS.open('/tx2/' + id, 'r'); + // .then(x => (console.error(x), x)) + .then(x => JSON.stringify(x)) - return stream; + FS.createDataFile('/', 'tx/' + id, Buffer.from(result, 'utf-8'), true, false) + const stream = FS.open('/tx/' + id, 'r') + return stream }, - async open(filename) { - const pathCategory = filename.split('/')[1]; - const id = filename.split('/')[2]; - console.log("JS: Opening ID: ", id); + async open (filename) { + const pathCategory = filename.split('/')[1] + const id = filename.split('/')[2] + console.log('JS: Opening ID: ', id) if (pathCategory === 'tx') { - FS.createPath('/', 'tx', true, false); + FS.createPath('/', 'tx', true, false) if (FS.analyzePath(filename).exists) { - var stream = FS.open(filename, 'r'); + const stream = FS.open(filename, 'r') if (stream.fd) return stream.fd return 0 } else { - const stream = await this.createTxHeader(id); - return stream.fd; - } - - } - if (pathCategory === 'tx2') { - FS.createPath('/', 'tx2', true, false); - if (FS.analyzePath(filename).exists) { - var stream = FS.open(filename, 'r'); - if (stream.fd) return stream.fd - return 0 - } else { - const stream = await this.createDataItemTxHeader(id); - if (stream.fd) return stream.fd - return 0 + const stream = await this.createTxHeader(id) + return stream.fd } } if (pathCategory === 'block') { - FS.createPath('/', 'block', true, false); + FS.createPath('/', 'block', true, false) if (FS.analyzePath(filename).exists) { - var stream = FS.open(filename, 'r'); + const stream = FS.open(filename, 'r') if (stream.fd) return stream.fd return 0 } else { - const stream = await this.createBlockHeader(id); - return stream.fd; + const stream = await this.createBlockHeader(id) + return stream.fd } } if (pathCategory === 'data') { if (FS.analyzePath(filename).exists) { - var stream = FS.open(filename, 'r'); + const stream = FS.open(filename, 'r') if (stream.fd) return stream.fd - console.log("JS: File not found: ", filename); - return 0; - } - else { - //console.log("JS: Open => Creating file: ", id); - const stream = await this.create(id); - //console.log("JS: Open => Created file: ", id, " fd: ", stream.fd); - return stream.fd; + console.log('JS: File not found: ', filename) + return 0 + } else { + // console.log("JS: Open => Creating file: ", id); + const stream = await this.create(id) + // console.log("JS: Open => Created file: ", id, " fd: ", stream.fd); + return stream.fd } - } - else if (pathCategory === 'headers') { - console.log("Header access not implemented yet."); - return 0; - } - else { - console.log("JS: Invalid path category: ", pathCategory); - return 0; + } else if (pathCategory === 'headers') { + console.log('Header access not implemented yet.') + return 0 + } else { + console.log('JS: Invalid path category: ', pathCategory) + return 0 } }, - async read(fd, raw_dst_ptr, raw_length) { + async read (fd, rawDstPtr, rawLength) { + // Note: The length and dstPtr are 53 bit integers in JS, so this _should_ be ok into a large memspace. + let toRead = Number(rawLength) + let dstPtr = Number(rawDstPtr) - // Note: The length and dst_ptr are 53 bit integers in JS, so this _should_ be ok into a large memspace. - var to_read = Number(raw_length) - var dst_ptr = Number(raw_dst_ptr) - - var stream = 0; - for (var i = 0; i < FS.streams.length; i++) { + let stream = 0 + for (let i = 0; i < FS.streams.length; i++) { if (FS.streams[i].fd === fd) { stream = FS.streams[i] } } // read block headers if (stream.path.includes('/block')) { - mod.HEAP8.set(stream.node.contents.subarray(0, to_read), dst_ptr); - return to_read; + mod.HEAP8.set(stream.node.contents.subarray(0, toRead), dstPtr) + return toRead } // read tx headers if (stream.path.includes('/tx')) { - mod.HEAP8.set(stream.node.contents.subarray(0, to_read), dst_ptr); - return to_read; + mod.HEAP8.set(stream.node.contents.subarray(0, toRead), dstPtr) + return toRead } // Satisfy what we can with the cache first - var bytes_read = this.readFromCache(stream, dst_ptr, to_read) - stream.position += bytes_read - stream.lastReadPosition = stream.position; - dst_ptr += bytes_read - to_read -= bytes_read + let bytesRead = this.readFromCache(stream, dstPtr, toRead) + stream.position += bytesRead + stream.lastReadPosition = stream.position + dstPtr += bytesRead + toRead -= bytesRead // Return if we have satisfied the request - if (to_read === 0) { - //console.log("WeaveDrive: Satisfied request with cache. Returning...") - return bytes_read + if (toRead === 0) { + // console.log("WeaveDrive: Satisfied request with cache. Returning...") + return bytesRead } - //console.log("WeaveDrive: Read from cache: ", bytes_read, " Remaining to read: ", to_read) + // console.log("WeaveDrive: Read from cache: ", bytesRead, " Remaining to read: ", toRead) - const chunk_download_sz = Math.max(to_read, CACHE_SZ) - const to = Math.min(stream.node.total_size, stream.position + chunk_download_sz); - //console.log("WeaveDrive: fd: ", fd, " Read length: ", to_read, " Reading ahead:", to - to_read - stream.position) + const chunkDownloadSz = Math.max(toRead, CACHE_SZ) + const to = Math.min(stream.node.total_size, stream.position + chunkDownloadSz) + // console.log("WeaveDrive: fd: ", fd, " Read length: ", toRead, " Reading ahead:", to - toRead - stream.position) // Fetch with streaming const response = await this.customFetch(`/${stream.node.name}`, { - method: "GET", - redirect: "follow", - headers: { "Range": `bytes=${stream.position}-${to}` } - }); + method: 'GET', + redirect: 'follow', + headers: { Range: `bytes=${stream.position}-${to}` } + }) const reader = response.body.getReader() - var bytes_until_cache = CHUNK_SZ - var bytes_until_notify = NOTIFY_SZ - var downloaded_bytes = 0 - var cache_chunks = [] + let bytesUntilCache = CHUNK_SZ + let bytesUntilNotify = NOTIFY_SZ + let downloadedBytes = 0 + let cacheChunks = [] try { while (true) { - const { done, value: chunk_bytes } = await reader.read(); - if (done) break; + const { done, value: chunkBytes } = await reader.read() + if (done) break // Update the number of downloaded bytes to be _all_, not just the write length - downloaded_bytes += chunk_bytes.length - bytes_until_cache -= chunk_bytes.length - bytes_until_notify -= chunk_bytes.length + downloadedBytes += chunkBytes.length + bytesUntilCache -= chunkBytes.length + bytesUntilNotify -= chunkBytes.length // Write bytes from the chunk and update the pointer if necessary - const write_length = Math.min(chunk_bytes.length, to_read); - if (write_length > 0) { - //console.log("WeaveDrive: Writing: ", write_length, " bytes to: ", dst_ptr) - mod.HEAP8.set(chunk_bytes.subarray(0, write_length), dst_ptr) - dst_ptr += write_length - bytes_read += write_length - stream.position += write_length - to_read -= write_length + const writeLength = Math.min(chunkBytes.length, toRead) + if (writeLength > 0) { + // console.log("WeaveDrive: Writing: ", writeLength, " bytes to: ", dstPtr) + mod.HEAP8.set(chunkBytes.subarray(0, writeLength), dstPtr) + dstPtr += writeLength + bytesRead += writeLength + stream.position += writeLength + toRead -= writeLength } - if (to_read == 0) { + if (toRead === 0) { // Add excess bytes to our cache - const chunk_to_cache = chunk_bytes.subarray(write_length) - //console.log("WeaveDrive: Cacheing excess: ", chunk_to_cache.length) - cache_chunks.push(chunk_to_cache) + const chunkToCache = chunkBytes.subarray(writeLength) + // console.log("WeaveDrive: Cacheing excess: ", chunkToCache.length) + cacheChunks.push(chunkToCache) } - if (bytes_until_cache <= 0) { - console.log("WeaveDrive: Chunk size reached. Compressing cache...") - stream.node.cache = this.addChunksToCache(stream.node.cache, cache_chunks) - cache_chunks = [] - bytes_until_cache = CHUNK_SZ + if (bytesUntilCache <= 0) { + console.log('WeaveDrive: Chunk size reached. Compressing cache...') + stream.node.cache = this.addChunksToCache(stream.node.cache, cacheChunks) + cacheChunks = [] + bytesUntilCache = CHUNK_SZ } - if (bytes_until_notify <= 0) { - console.log("WeaveDrive: Downloaded: ", downloaded_bytes / stream.node.total_size * 100, "%") - bytes_until_notify = NOTIFY_SZ + if (bytesUntilNotify <= 0) { + console.log('WeaveDrive: Downloaded: ', downloadedBytes / stream.node.total_size * 100, '%') + bytesUntilNotify = NOTIFY_SZ } } } catch (error) { - console.error("WeaveDrive: Error reading the stream: ", error) + console.error('WeaveDrive: Error reading the stream: ', error) } finally { reader.releaseLock() } // If we have no cache, or we have not satisfied the full request, we need to download the rest // Rebuild the cache from the new cache chunks - stream.node.cache = this.addChunksToCache(stream.node.cache, cache_chunks) + stream.node.cache = this.addChunksToCache(stream.node.cache, cacheChunks) // Update the last read position stream.lastReadPosition = stream.position - return bytes_read + return bytesRead }, - close(fd) { - var stream = 0; - for (var i = 0; i < FS.streams.length; i++) { + close (fd) { + let stream = 0 + for (let i = 0; i < FS.streams.length; i++) { if (FS.streams[i].fd === fd) { stream = FS.streams[i] } @@ -403,43 +296,43 @@ module.exports = function weaveDrive(mod, FS) { }, // Readahead cache functions - readFromCache(stream, dst_ptr, length) { + readFromCache (stream, dstPtr, length) { // Check if the cache has been invalidated by a seek if (stream.lastReadPosition !== stream.position) { - //console.log("WeaveDrive: Invalidating cache for fd: ", stream.fd, " Current pos: ", stream.position, " Last read pos: ", stream.lastReadPosition) + // console.log("WeaveDrive: Invalidating cache for fd: ", stream.fd, " Current pos: ", stream.position, " Last read pos: ", stream.lastReadPosition) stream.node.cache = new Uint8Array(0) return 0 } // Calculate the bytes of the request that can be satisfied with the cache - var cache_part_length = Math.min(length, stream.node.cache.length) - var cache_part = stream.node.cache.subarray(0, cache_part_length) - mod.HEAP8.set(cache_part, dst_ptr) + const cachePartLength = Math.min(length, stream.node.cache.length) + const cachePart = stream.node.cache.subarray(0, cachePartLength) + mod.HEAP8.set(cachePart, dstPtr) // Set the new cache to the remainder of the unused cache and update pointers - stream.node.cache = stream.node.cache.subarray(cache_part_length) + stream.node.cache = stream.node.cache.subarray(cachePartLength) - return cache_part_length + return cachePartLength }, - addChunksToCache(old_cache, chunks) { + addChunksToCache (oldCache, chunks) { // Make a new cache array of the old cache length + the sum of the chunk lengths, capped by the max cache size - var new_cache_length = Math.min(old_cache.length + chunks.reduce((acc, chunk) => acc + chunk.length, 0), CACHE_SZ) - var new_cache = new Uint8Array(new_cache_length) + const newCacheLength = Math.min(oldCache.length + chunks.reduce((acc, chunk) => acc + chunk.length, 0), CACHE_SZ) + const newCache = new Uint8Array(newCacheLength) // Copy the old cache to the new cache - new_cache.set(old_cache, 0) + newCache.set(oldCache, 0) // Load the cache chunks into the new cache - var current_offset = old_cache.length; - for (let chunk of chunks) { - if (current_offset < new_cache_length) { - new_cache.set(chunk.subarray(0, new_cache_length - current_offset), current_offset); - current_offset += chunk.length; + let currentOffset = oldCache.length + for (const chunk of chunks) { + if (currentOffset < newCacheLength) { + newCache.set(chunk.subarray(0, newCacheLength - currentOffset), currentOffset) + currentOffset += chunk.length } } - return new_cache + return newCache }, // General helpder functions - async checkAdmissible(ID) { - if (mod.mode && mod.mode == "test") { + async checkAdmissible (ID) { + if (mod.mode && mod.mode === 'test') { // CAUTION: If the module is initiated with `mode = test` we don't check availability. return true } @@ -447,42 +340,38 @@ module.exports = function weaveDrive(mod, FS) { // Check if we are attempting to load the On-Boot id, if so allow it // this was added for AOP 6 Boot loader See: https://github.com/permaweb/aos/issues/342 const bootTag = this.getTagValue('On-Boot', mod.spawn.tags) - if (bootTag && (bootTag === ID)) return true; + if (bootTag && (bootTag === ID)) return true // Check that this module or process set the WeaveDrive tag on spawn const blockHeight = mod.blockHeight - const moduleExtensions = this.getTagValues("Extension", mod.module.tags) - const moduleHasWeaveDrive = moduleExtensions.includes("WeaveDrive") - const processExtensions = this.getTagValues("Extension", mod.spawn.tags) - const processHasWeaveDrive = moduleHasWeaveDrive || processExtensions.includes("WeaveDrive") + const moduleExtensions = this.getTagValues('Extension', mod.module.tags) + const moduleHasWeaveDrive = moduleExtensions.includes('WeaveDrive') + const processExtensions = this.getTagValues('Extension', mod.spawn.tags) + const processHasWeaveDrive = moduleHasWeaveDrive || processExtensions.includes('WeaveDrive') if (!processHasWeaveDrive) { - console.log("WeaveDrive: Process tried to call WeaveDrive, but extension not set!") + console.log('WeaveDrive: Process tried to call WeaveDrive, but extension not set!') return false } - const modes = ["Assignments", "Individual", "Library"] + const modes = ['Assignments', 'Individual', 'Library'] // Get the Availability-Type from the spawned process's Module or Process item // First check the module for its defaults const moduleAvailabilityType = this.getTagValue('Availability-Type', mod.module.tags) - const moduleMode = moduleAvailabilityType - ? moduleAvailabilityType - : "Assignments" // Default to assignments + const moduleMode = moduleAvailabilityType || 'Assignments' // Default to assignments // Now check the process's spawn item. These settings override Module item settings. const processAvailabilityType = this.getTagValue('Availability-Type', mod.spawn.tags) - const processMode = processAvailabilityType - ? processAvailabilityType - : moduleMode + const processMode = processAvailabilityType || moduleMode if (!modes.includes(processMode)) { - throw `Unsupported WeaveDrive mode: ${processMode}` + throw Error(`Unsupported WeaveDrive mode: ${processMode}`) } const attestors = this.serializeStringArr( [ this.getTagValue('Scheduler', mod.spawn.tags), - ...this.getTagValues("Attestor", mod.spawn.tags) + ...this.getTagValues('Attestor', mod.spawn.tags) ].filter(t => !!t) ) @@ -515,7 +404,7 @@ module.exports = function weaveDrive(mod, FS) { return true } - if (processMode == "Individual") { + if (processMode === 'Individual') { const individualsHaveID = await this.queryHasResult( `query { transactions( @@ -550,21 +439,21 @@ module.exports = function weaveDrive(mod, FS) { // execution failed on this message. The CU must also not continue to execute further // messages on this process. Attesting to them would be slashable, as the state would // be incorrect. - if (processMode == "Library") { - throw "This WeaveDrive implementation does not support Library attestations yet!" + if (processMode === 'Library') { + throw Error('This WeaveDrive implementation does not support Library attestations yet!') } return false }, serializeStringArr (arr = []) { - return`[${arr.map((s) => `"${s}"`).join(', ')}]` + return `[${arr.map((s) => `"${s}"`).join(', ')}]` }, - getTagValues(key, tags) { - var values = [] - for (i = 0; i < tags.length; i++) { - if (tags[i].name == key) { + getTagValues (key, tags) { + const values = [] + for (let i = 0; i < tags.length; i++) { + if (tags[i].name === key) { values.push(tags[i].value) } } @@ -576,37 +465,20 @@ module.exports = function weaveDrive(mod, FS) { return values.pop() }, - async queryHasResult(query, variables) { + async queryHasResult (query, variables) { const json = await this.gqlQuery(query, variables) .then((res) => res.json()) return !!json?.data?.transactions?.edges?.length }, - - async gqlExists() { - const query = `query { - transactions( - first: 1 - ) { - pageInfo { - hasNextPage - } - } - } - ` - - const gqlExists = await this.gqlQuery(query, {}).then((res) => res.ok) - return gqlExists - }, - - async gqlQuery(query, variables) { + async gqlQuery (query, variables) { const options = { method: 'POST', body: JSON.stringify({ query, variables }), - headers: { 'Content-Type': 'application/json'} + headers: { 'Content-Type': 'application/json' } } return this.customFetch('graphql', options) } } -} \ No newline at end of file +} diff --git a/extensions/weavedrive/test/index.test.js b/extensions/weavedrive/test/index.test.js index e48aa76b..4eded96b 100644 --- a/extensions/weavedrive/test/index.test.js +++ b/extensions/weavedrive/test/index.test.js @@ -9,8 +9,8 @@ const bootLoaderWasm = fs.readFileSync('./bootloader.wasm') let memory = null const Module = { - Id: "MODULE", - Owner: "OWNER", + Id: 'MODULE', + Owner: 'OWNER', Tags: [ { name: 'Data-Protocol', value: 'ao' }, { name: 'Variant', value: 'ao.TN.1' }, @@ -43,7 +43,7 @@ const Msg = { { name: 'Type', value: 'Message' }, { name: 'Action', value: 'Eval' } ], - "Block-Height": 1000, + 'Block-Height': 1000, Timestamp: Date.now() } @@ -51,7 +51,7 @@ const options = { format: 'wasm64-unknown-emscripten-draft_2024_02_15', WeaveDrive: weaveDrive, ARWEAVE: 'https://arweave.net', - mode: "test", + mode: 'test', blockHeight: 1000, spawn: { tags: Process.Tags @@ -167,24 +167,25 @@ describe('Assignments mode', () => { { name: 'Authority', value: 'PROCESS' }, { name: 'Availability-Type', value: mode } ], - Data: `Test = 1`, + Data: 'Test = 1', From: 'PROCESS', Module: 'MODULE', - "Block-Height": 4567, + 'Block-Height': 4567, Timestamp: Date.now() } test('read tx attested by Scheduler', async () => { - ProcessSchedulerAttested = { + const ProcessSchedulerAttested = { ...ProcessAssignmentsMode, Tags: [ ...ProcessAssignmentsMode.Tags, - { name: 'Scheduler', value: 'kdUCABg56Jroco1kMwfF-YIjah9wBbZ1BhyOnwLwOY0' }, + { name: 'Scheduler', value: 'kdUCABg56Jroco1kMwfF-YIjah9wBbZ1BhyOnwLwOY0' } ] } const handle = await AoLoader(wasm, { - ...options, spawn: { + ...options, + spawn: { id: ProcessSchedulerAttested.Id, owner: ProcessSchedulerAttested.Owner, tags: ProcessSchedulerAttested.Tags @@ -201,7 +202,7 @@ describe('Assignments mode', () => { return drive.getData("${TX_ID_TO_LOAD}") ` }, { Process: ProcessSchedulerAttested, Module }) - + assert.equal(result.Output.data, 'hello from attested') }) @@ -211,7 +212,7 @@ describe('Assignments mode', () => { Tags: [ ...ProcessAssignmentsMode.Tags, { name: 'Scheduler', value: 'something-else' }, - { name: 'Attestor', value: 'kdUCABg56Jroco1kMwfF-YIjah9wBbZ1BhyOnwLwOY0' }, + { name: 'Attestor', value: 'kdUCABg56Jroco1kMwfF-YIjah9wBbZ1BhyOnwLwOY0' } ] } @@ -234,7 +235,7 @@ describe('Assignments mode', () => { return drive.getData("${TX_ID_TO_LOAD}") ` }, { Process: ProcessAttestorAttested, Module }) - + assert.equal(result.Output.data, 'hello from attested') }) }) @@ -255,12 +256,12 @@ describe('Individual Mode', () => { { name: 'Extension', value: 'WeaveDrive' }, { name: 'Module', value: 'MODULE' }, { name: 'Authority', value: 'PROCESS' }, - { name: 'Availability-Type', value: mode }, + { name: 'Availability-Type', value: mode } ], - Data: `Test = 1`, + Data: 'Test = 1', From: 'PROCESS', Module: 'MODULE', - "Block-Height": 4567, + 'Block-Height': 4567, Timestamp: Date.now() } @@ -269,7 +270,7 @@ describe('Individual Mode', () => { ...ProcessIndividualMode, Tags: [ ...ProcessIndividualMode.Tags, - { name: 'Scheduler', value: 'kdUCABg56Jroco1kMwfF-YIjah9wBbZ1BhyOnwLwOY0' }, + { name: 'Scheduler', value: 'kdUCABg56Jroco1kMwfF-YIjah9wBbZ1BhyOnwLwOY0' } ] } @@ -292,14 +293,14 @@ describe('Individual Mode', () => { return drive.getData("${TX_ID_TO_LOAD}") ` }, { Process: ProcessSchedulerAttested, Module }) - + assert.equal(result.Output.data, '1234') }) }) // test weavedrive feature of accepting multiple gateways describe('multi-url', () => { - const urls = 'https://arweave.net/does-not-exist,https://g8way.io' + const urls = 'https://arweave.net/does-not-exist,https://g8way.io' test('read block', async () => { const handle = await AoLoader(wasm, { ...options, @@ -314,8 +315,7 @@ describe('multi-url', () => { memory = result.Memory assert.equal(result.Output.data, '20') }) - - + test('read tx', async () => { const handle = await AoLoader(wasm, { ...options, @@ -338,78 +338,6 @@ describe('multi-url', () => { }) }) -test('read data item tx', async () => { - const handle = await AoLoader(wasm, options) - const result = await handle(memory, { - ...Msg, - Data: ` -local results = {} -local drive = require('WeaveDrive') -local result = drive.getDataItem('Jy_AFHfmxVsrtJoxeJZfq9dx_ES730a7uO2lyYtO6uU') - -return result - ` - }, { Process, Module }) - const data = JSON.parse(result.Output.data) - assert.equal(data.format, 3) - assert.equal(data.id, 'Jy_AFHfmxVsrtJoxeJZfq9dx_ES730a7uO2lyYtO6uU') - assert.equal(data.block.height, 1290333) -}) - -test('data item with chinese characters', async () => { - const chineseCharacterTx = 'Xg82mcAfQNgJetk_kSKqMkwYDfoMJLGw8fdheU3evWk' - const handle = await AoLoader(wasm, options) - const result = await handle(memory, { - ...Msg, - Data: ` - local data = require('WeaveDrive').getDataItem('${chineseCharacterTx}') - local json = require('json') - local decoded = json.decode(data) - return data - ` - }, { Process, Module }) - memory = result.Memory - const data = JSON.parse(result.Output.data) - const description = data.tags.find(tag => tag.name === 'Description')?.value - assert.equal(description, "Smartweave是不是要转到AO了？\n本文档介绍原子资产SmartWeave合约的标准接口。原子资产是SmartWeave合约及其在发布到Arweave的同一事务中初始化的数据。SmartWeave合约和数据共享一个交易ID。") -}) - -test('read data item tx, no result', async () => { - const handle = await AoLoader(wasm, options) - const result = await handle(memory, { - ...Msg, - Data: ` -local address = 'foo-address' -local results = {} -local drive = require('WeaveDrive') -local result = drive.getDataItem(address) - -return result - ` - }, { Process, Module }) - - memory = result.Memory - assert.equal(result.Output.data, '') -}) - -test('read data item tx, no gql', async () => { - const handle = await AoLoader(wasm, { - ...options, - ARWEAVE: 'https://example.com' - }) - const result = await handle(memory, { - ...Msg, - Data: ` -local results = {} -local drive = require('WeaveDrive') -local result = drive.getDataItem('Jy_AFHfmxVsrtJoxeJZfq9dx_ES730a7uO2lyYtO6uU') -return result - ` - }, { Process, Module }) - memory = result.Memory - assert.equal(result.Output.data, '') -}) - test('boot loader set to Data', async function () { /** * The Process is also the first message when aop 6 boot loader @@ -434,7 +362,7 @@ test('boot loader set to Data', async function () { `, From: 'PROCESS', Module: 'MODULE', - "Block-Height": 1234, + 'Block-Height': 1234, Timestamp: Date.now() } @@ -467,7 +395,7 @@ test('boot loader set to tx id', async function () { `, From: 'PROCESS', Module: 'MODULE', - "Block-Height": 4567, + 'Block-Height': 4567, Timestamp: Date.now() } @@ -478,7 +406,7 @@ test('boot loader set to tx id', async function () { owner: ProcessBootLoaderTx.Owner, tags: ProcessBootLoaderTx.Tags }, - mode: null + mode: null } const handle = await AoLoader(bootLoaderWasm, optionsBootLoaderTx) From 477786f07c1d12f6b69db9f5c1058ace332b8244 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Mon, 25 Nov 2024 11:33:37 -0500 Subject: [PATCH 2/3] fix(weavedrive): remove tx2 data items --- extensions/weavedrive/client/main.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/weavedrive/client/main.lua b/extensions/weavedrive/client/main.lua index ca9dabdb..1e300b73 100644 --- a/extensions/weavedrive/client/main.lua +++ b/extensions/weavedrive/client/main.lua @@ -26,7 +26,7 @@ function drive.getTx(txId) end function drive.getData(txId) - local file = io.open("/data/" .. txId) + local file = io.open("tx2/" .. txId) if not file then return nil, "File not found!" end From ae26cf4096118163ddd77460ec8d3415b688b5c8 Mon Sep 17 00:00:00 2001 From: Jack Frain Date: Mon, 25 Nov 2024 11:34:46 -0500 Subject: [PATCH 3/3] fix(weavedrive): remove tx2 data items --- extensions/weavedrive/client/main.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/weavedrive/client/main.lua b/extensions/weavedrive/client/main.lua index 1e300b73..ca9dabdb 100644 --- a/extensions/weavedrive/client/main.lua +++ b/extensions/weavedrive/client/main.lua @@ -26,7 +26,7 @@ function drive.getTx(txId) end function drive.getData(txId) - local file = io.open("tx2/" .. txId) + local file = io.open("/data/" .. txId) if not file then return nil, "File not found!" end