From b30b57dd2615c91bb808537971987bfe443266c0 Mon Sep 17 00:00:00 2001 From: vshcherb Date: Mon, 4 Nov 2024 16:14:32 +0200 Subject: [PATCH 1/2] Read cache prototype --- .../osmand/binary/BinaryMapIndexReader.java | 41 +++++++++++++++---- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java b/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java index e13d1679a16..f237df83ea2 100644 --- a/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java +++ b/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java @@ -987,13 +987,7 @@ public List searchMapIndex(SearchRequest req.right || tree.top > req.bottom || tree.bottom < req.top) { - continue; - } - codedIS.seek(tree.filePointer); - long oldLimit = codedIS.pushLimitLong((long) tree.length); - searchMapTreeBounds(tree, index, req, foundSubtrees); - codedIS.popLimit(oldLimit); + searchMapTreeBoundsCache(tree, index, req, foundSubtrees); } Collections.sort(foundSubtrees, new Comparator() { @@ -1098,11 +1092,32 @@ protected void readMapDataBlocks(SearchRequest req, MapTree } } + + + protected void searchMapTreeBoundsCache(MapTree current, MapTree parent, + SearchRequest req, List foundSubtrees) throws IOException { + if (current.right < req.left || current.left > req.right || current.top > req.bottom || current.bottom < req.top) { + return; + } + if (current.children != null) { + for (MapTree ch : current.children) { + searchMapTreeBoundsCache(ch, current, req, foundSubtrees); + } + } else { + codedIS.seek(current.filePointer); + long oldLimit = codedIS.pushLimitLong((long) current.length); + searchMapTreeBounds(current, parent, req, foundSubtrees); + codedIS.popLimit(oldLimit); + } + } protected void searchMapTreeBounds(MapTree current, MapTree parent, SearchRequest req, List foundSubtrees) throws IOException { int init = 0; req.numberOfReadSubtrees++; + if (current.depth < 7) { + current.children = new ArrayList<>(); + } while (true) { if (req.isCancelled()) { return; @@ -1152,16 +1167,21 @@ protected void searchMapTreeBounds(MapTree current, MapTree parent, break; case MapDataBox.BOXES_FIELD_NUMBER : // left, ... already initialized + MapTree child = new MapTree(); child.length = readInt(); + child.depth = current.depth + 1; child.filePointer = codedIS.getTotalBytesRead(); long oldLimit = codedIS.pushLimitLong((long) child.length); - if(current.ocean != null ){ + if (current.ocean != null) { child.ocean = current.ocean; } searchMapTreeBounds(child, current, req, foundSubtrees); codedIS.popLimit(oldLimit); codedIS.seek(child.filePointer + child.length); + if(current.children != null) { + current.children.add(child); + } break; default: skipUnknownField(t); @@ -2232,12 +2252,15 @@ private static class MapTree { long mapDataBlock = 0; Boolean ocean = null; + int depth; int left = 0; int right = 0; int top = 0; int bottom = 0; - + List children = null; + + public int getLeft() { return left; } From ec8882e94e6b3adb3450414c5bd21484457fd0f5 Mon Sep 17 00:00:00 2001 From: vshcherb Date: Mon, 18 Nov 2024 13:47:52 +0100 Subject: [PATCH 2/2] Fix compilation --- .../com/google/protobuf/CodedInputStream.java | 16 ++++++++++++++++ .../net/osmand/binary/BinaryMapIndexReader.java | 10 +++++----- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/OsmAnd-java/src/main/java/com/google/protobuf/CodedInputStream.java b/OsmAnd-java/src/main/java/com/google/protobuf/CodedInputStream.java index e7f2f273797..77eaf32d3fe 100644 --- a/OsmAnd-java/src/main/java/com/google/protobuf/CodedInputStream.java +++ b/OsmAnd-java/src/main/java/com/google/protobuf/CodedInputStream.java @@ -564,6 +564,10 @@ public static long decodeZigZag64(final long n) { /** See setSizeLimit() */ // ! osmand change ! private long sizeLimit = DEFAULT_SIZE_LIMIT; + // FIXME this part not ready to be merged to master (delete) + private boolean READ_SEEK; + public static int READ_BYTES; + public static int READ_BLOCKS; private static final int DEFAULT_RECURSION_LIMIT = 64; // ! osmand change ! @@ -774,6 +778,11 @@ private boolean refillBuffer(final boolean mustSucceed) throws IOException { bufferSize = (int) Math.min(remain, buffer.length); if(bufferSize > 0) { raf.readFully(buffer, 0, bufferSize); + if (READ_SEEK) { + READ_BLOCKS++; + READ_SEEK = false; + } + READ_BYTES += bufferSize; } else { bufferSize = -1; } @@ -896,6 +905,11 @@ public byte[] readRawBytes(final int size) throws IOException { final int n; // osmand change if(raf != null) { + READ_BYTES += chunk.length - pos; + if (READ_SEEK) { + READ_BLOCKS++; + READ_SEEK = false; + } raf.readFully(chunk, pos, chunk.length - pos); n = chunk.length - pos; } else { @@ -960,6 +974,7 @@ public void skipRawBytes(final long size) throws IOException { if(raf != null) { bufferPos = 0; bufferSize = 0; + READ_SEEK = true; raf.seek(raf.getFilePointer() + (size - pos)); totalBytesRetired = raf.getFilePointer(); } else { @@ -985,6 +1000,7 @@ public void seek(long pointer) throws IOException { } bufferPos = (int) (pointer - totalBytesRetired); } else { + READ_SEEK = true; totalBytesRetired = pointer; bufferSizeAfterLimit = 0; raf.seek(pointer); diff --git a/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java b/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java index f237df83ea2..3bf79cb71fe 100644 --- a/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java +++ b/OsmAnd-java/src/main/java/net/osmand/binary/BinaryMapIndexReader.java @@ -92,6 +92,7 @@ public class BinaryMapIndexReader { public static final int SHIFT_COORDINATES = 5; public static final int LABEL_ZOOM_ENCODE = 31 - SHIFT_COORDINATES; private final static Log log = PlatformUtil.getLog(BinaryMapIndexReader.class); + public static int DEPTH_CACHE = 7; public static boolean READ_STATS = false; public static final SearchPoiTypeFilter ACCEPT_ALL_POI_TYPE_FILTER = new SearchPoiTypeFilter() { @Override @@ -1115,9 +1116,6 @@ protected void searchMapTreeBounds(MapTree current, MapTree parent, SearchRequest req, List foundSubtrees) throws IOException { int init = 0; req.numberOfReadSubtrees++; - if (current.depth < 7) { - current.children = new ArrayList<>(); - } while (true) { if (req.isCancelled()) { return; @@ -1167,7 +1165,9 @@ protected void searchMapTreeBounds(MapTree current, MapTree parent, break; case MapDataBox.BOXES_FIELD_NUMBER : // left, ... already initialized - + if (current.depth < DEPTH_CACHE && current.children == null) { + current.children = new ArrayList<>(); + } MapTree child = new MapTree(); child.length = readInt(); child.depth = current.depth + 1; @@ -1179,7 +1179,7 @@ protected void searchMapTreeBounds(MapTree current, MapTree parent, searchMapTreeBounds(child, current, req, foundSubtrees); codedIS.popLimit(oldLimit); codedIS.seek(child.filePointer + child.length); - if(current.children != null) { + if (current.children != null) { current.children.add(child); } break;