Proof of concept: parse head advertisement

cid-vs-multihash.js

@@ -0,0 +1,18 @@
+import { CID } from 'multiformats/cid'
+
+/*
+HUMAN READABLE CID
+base32    - cidv1   - raw        - (sha2-256 : 256 : FAE8D07F50DBBCD3FDD65610ADCD6C3DF2171B31AE97041B1E222506E78DFD94)
+MULTIBASE - VERSION - MULTICODEC - MULTIHASH (NAME : SIZE : DIGEST IN HEX)
+*/
+const MY_CID = 'bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq'
+console.log('CID string', MY_CID)
+
+// Found via https://cid.contact/cid/bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq
+const MULTIHASH = 'EiD66NB/UNu80/3WVhCtzWw98hcbMa6XBBseIiUG5439lA=='
+console.log('IPNI Multihash (base64)', MULTIHASH)
+console.log('IPNI multihash (hex)', Buffer.from(MULTIHASH, 'base64').toString('hex'))
+
+const cid = CID.parse(MY_CID)
+console.log('CID multihash (hex) ', Buffer.from(cid.multihash.bytes).toString('hex'))
+console.log('CID codec', cid.code.toString(16))

package.json

@@ -16,5 +16,9 @@
   "bugs": {
     "url": "https://github.com/filecoin-station/piece-indexer/issues"
   },
-  "homepage": "https://github.com/filecoin-station/piece-indexer#readme"
+  "homepage": "https://github.com/filecoin-station/piece-indexer#readme",
+  "dependencies": {
+    "@multiformats/multiaddr-to-uri": "^10.1.0",
+    "multiformats": "^13.1.3"
+  }
 }

poc.js

@@ -0,0 +1,83 @@
+import assert from 'node:assert'
+import { multiaddrToUri } from '@multiformats/multiaddr-to-uri'
+
+// IPNI specs: https://github.com/ipni/specs
+
+// TODO: list all index providers
+// https://cid.contact/providers
+
+// https://github.com/filecoin-station/frisbii-on-fly
+// This index provider does not set Graphsync metadata
+// const providerPeerId = '12D3KooWHge6fZmx6fMsizP9YYpbJPNZjiWz7Ye1WLCvmj6VTnjq'
+//
+// Doesn't work - cannot fetch Entries CID
+// Error: Cannot GET http://183.60.90.198:3104/ipni/v1/ad/bafkreehdwdcefgh4dqkjv67uzcmw7oje: 500
+// unable to load data for cid
+// const providerPeerId = '12D3KooWFpNqyFpqujkMXeKrbtasvKkbUSL8ipN5vXtNyfVo7n4f'
+
+// This one works:
+const providerPeerId = '12D3KooWDYiKtcxTrjNFtR6UqKRkJpESYHmmFznQAAkDX2ZHQ49t'
+
+// Get the latest announcement
+const res = await fetch(`https://cid.contact/providers/${encodeURIComponent(providerPeerId)}`)
+assert(res.ok)
+
+/** @type {{
+AddrInfo:{ID:string,Addrs:string[]},
+LastAdvertisement:{"/":string},
+LastAdvertisementTime: string,
+Publisher:{ID:string,Addrs:string[]},
+FrozenAt:null
+ * }}
+ */
+const providerMetadata = await res.json()
+
+console.log('Provider metadata:', providerMetadata)
+const providerBaseUrl = multiaddrToUri(providerMetadata.Publisher.Addrs[0], { assumeHttp: false })
+if (!providerBaseUrl.startsWith('http')) {
+  throw new Error(`Unsupported URI scheme: ${providerBaseUrl}`)
+}
+
+// TODO: handle libp2p index providers, e.g.
+// Publisher: {
+//   ID: '12D3KooWCmLYzfYU2fWVnWJDuEkjt7d8pq7PHYAkXoUFVCPogTA9',
+//   Addrs: [ '/ip4/103.9.208.54/tcp/48080' ]
+// }
+
+/** @type {{
+  Addresses: string[],
+  ContextID: { '/': { bytes: string } },,
+  Entries: {
+    '/': string
+  },
+  IsRm: false,
+  Metadata: { '/': { bytes: string } },
+  Provider: string
+  Signature: {
+    '/': {
+      bytes: string
+    }
+  }
+ }} */
+const head = await fetchCid(providerBaseUrl, providerMetadata.LastAdvertisement['/'])
+console.log('HEAD', head)
+
+const entries = await fetchCid(providerBaseUrl, head.Entries['/'])
+// console.log('ENTRIES', entries.Entries)
+
+const entryHash = entries.Entries[0]['/'].bytes
+console.log('FIRST ENTRY:', entryHash)
+// const multihash = Buffer.from(entryHash, 'base64')
+// const payloadCid = CID.parse('m' + entryHash, base64.decoder)
+// console.log('PAYLOAD CID', payloadCid)
+
+async function fetchCid (providerBaseUrl, adCid) {
+  const url = new URL(adCid, new URL('/ipni/v1/ad/_', providerBaseUrl))
+  console.log('fetching %s', url)
+  const res = await fetch(url)
+  console.log('status', res.status)
+  if (!res.ok) {
+    throw new Error(`Cannot GET ${url}: ${res.status}\n${await res.text()}`)
+  }
+  return await res.json()
+}