From 585a3492be6ca90784b586c4b8ff562b09c863e1 Mon Sep 17 00:00:00 2001 From: Eduardo Ramirez Date: Tue, 8 Oct 2024 16:56:26 -0700 Subject: [PATCH 1/3] with optimization ish --- .../objectmapper/HollowListTypeMapper.java | 16 +- .../objectmapper/HollowMapTypeMapper.java | 22 ++- .../objectmapper/HollowObjectMapper.java | 14 +- .../objectmapper/HollowObjectTypeMapper.java | 113 ++++++----- .../objectmapper/HollowSetTypeMapper.java | 13 +- .../write/objectmapper/HollowTypeMapper.java | 6 +- .../flatrecords/FlatRecordOrdinalReader.java | 176 +++++++++++++++++- .../FlatRecordTraversalListNode.java | 5 + .../traversal/FlatRecordTraversalMapNode.java | 5 + .../traversal/FlatRecordTraversalNode.java | 2 + .../FlatRecordTraversalObjectNode.java | 5 + .../traversal/FlatRecordTraversalSetNode.java | 5 + 12 files changed, 299 insertions(+), 83 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowListTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowListTypeMapper.java index 1c84ecc45..c959bfdf2 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowListTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowListTypeMapper.java @@ -24,9 +24,8 @@ import com.netflix.hollow.core.write.HollowListWriteRecord; import com.netflix.hollow.core.write.HollowTypeWriteState; import com.netflix.hollow.core.write.HollowWriteRecord; +import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordOrdinalReader; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalListNode; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalNode; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -126,14 +125,15 @@ protected Object parseHollowRecord(HollowRecord record) { } @Override - protected Object parseFlatRecord(FlatRecordTraversalNode node) { - List collection = new ArrayList<>(); - - for (FlatRecordTraversalNode elementNode : (FlatRecordTraversalListNode) node) { - Object element = elementMapper.parseFlatRecord(elementNode); + protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { + FlatRecordOrdinalReader.Offset offset = reader.getOffsetAtDataStartOf(ordinal); + int size = reader.readSize(offset); + List collection = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + int elementOrdinal = reader.readListElementOrdinal(offset); + Object element = elementMapper.parseFlatRecord(reader, elementOrdinal); collection.add(element); } - return collection; } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java index 6de0da6e7..26bb2a38b 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java @@ -25,9 +25,8 @@ import com.netflix.hollow.core.write.HollowTypeWriteState; import com.netflix.hollow.core.write.HollowWriteRecord; import com.netflix.hollow.core.write.HollowWriteStateEngine; +import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordOrdinalReader; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalMapNode; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalNode; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -140,13 +139,18 @@ protected Object parseHollowRecord(HollowRecord record) { } @Override - protected Object parseFlatRecord(FlatRecordTraversalNode node) { - FlatRecordTraversalMapNode mapNode = (FlatRecordTraversalMapNode) node; - Map collection = new HashMap<>(); - - for (Map.Entry entry : mapNode.entrySet()) { - Object key = keyMapper.parseFlatRecord(entry.getKey()); - Object value = valueMapper.parseFlatRecord(entry.getValue()); + protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { + FlatRecordOrdinalReader.Offset offset = reader.getOffsetAtDataStartOf(ordinal); + int size = reader.readSize(offset); + Map collection = new HashMap<>(size); + int keyOrdinal = 0; + for (int i = 0; i < size; i++) { + long keyOrdinalDeltaAndValueOrdinal = reader.readMapKeyOrdinalDeltaAndValueOrdinal(offset); + keyOrdinal += (int) (keyOrdinalDeltaAndValueOrdinal >>> 32); + int valueOrdinal = (int) keyOrdinalDeltaAndValueOrdinal; + + Object key = keyMapper.parseFlatRecord(reader, keyOrdinal); + Object value = valueMapper.parseFlatRecord(reader, valueOrdinal); collection.put(key, value); } return collection; diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapper.java index 6a1a04b70..b4fabae30 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapper.java @@ -20,6 +20,7 @@ import com.netflix.hollow.core.schema.HollowSchema; import com.netflix.hollow.core.write.HollowWriteStateEngine; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecord; +import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordOrdinalReader; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordReader; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter; import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalNode; @@ -100,13 +101,20 @@ public T readFlat(FlatRecordTraversalNode node) { if (typeMapper == null) { throw new IllegalArgumentException("No type mapper found for schema " + schemaName); } - Object obj = typeMapper.parseFlatRecord(node); + Object obj = typeMapper.parseFlatRecord(node.getReader(), node.getOrdinal()); return (T) obj; } public T readFlat(FlatRecord record) { - FlatRecordTraversalNode node = new FlatRecordTraversalObjectNode(record); - return readFlat(node); + FlatRecordOrdinalReader reader = new FlatRecordOrdinalReader(record); + int ordinal = reader.getOrdinalCount() - 1; + HollowSchema schema = reader.readSchema(ordinal); + HollowTypeMapper typeMapper = typeMappers.get(schema.getName()); + if (typeMapper == null) { + throw new IllegalArgumentException("No type mapper found for schema " + schema.getName()); + } + Object obj = typeMapper.parseFlatRecord(reader, ordinal); + return (T) obj; } /** diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java index ecbac4fdc..34935d29d 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java @@ -26,6 +26,7 @@ import com.netflix.hollow.core.write.HollowObjectWriteRecord; import com.netflix.hollow.core.write.HollowTypeWriteState; import com.netflix.hollow.core.write.HollowWriteRecord; +import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordOrdinalReader; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -37,8 +38,6 @@ import java.util.List; import java.util.Set; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalNode; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalObjectNode; import sun.misc.Unsafe; @SuppressWarnings("restriction") @@ -244,10 +243,10 @@ protected Object parseHollowRecord(HollowRecord record) { } @Override - protected Object parseFlatRecord(FlatRecordTraversalNode node) { + protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { + FlatRecordOrdinalReader.Offset offset = reader.getOffsetOf(ordinal); try { - FlatRecordTraversalObjectNode objectNode = (FlatRecordTraversalObjectNode) node; - HollowObjectSchema flatRecordSchema = objectNode.getSchema(); + HollowObjectSchema flatRecordSchema = (HollowObjectSchema) reader.readSchema(offset); Object obj = null; if (BOXED_WRAPPERS.contains(clazz)) { @@ -256,7 +255,9 @@ protected Object parseFlatRecord(FlatRecordTraversalNode node) { for (int i = 0; i < flatRecordSchema.numFields(); i++) { int posInPojoSchema = schema.getPosition(flatRecordSchema.getFieldName(i)); if (posInPojoSchema != -1) { - obj = mappedFields.get(posInPojoSchema).parseBoxedWrapper(objectNode); + obj = mappedFields.get(posInPojoSchema).parseBoxedWrapper(reader, offset); + } else { + reader.skipField(offset, schema.getFieldType(i)); } } } else if (clazz.isEnum()) { @@ -267,7 +268,9 @@ protected Object parseFlatRecord(FlatRecordTraversalNode node) { String fieldName = flatRecordSchema.getFieldName(i); int posInPojoSchema = schema.getPosition(fieldName); if (fieldName.equals(MappedFieldType.ENUM_NAME.getSpecialFieldName()) && posInPojoSchema != -1) { - obj = mappedFields.get(posInPojoSchema).parseBoxedWrapper(objectNode); + obj = mappedFields.get(posInPojoSchema).parseBoxedWrapper(reader, offset); + } else { + reader.skipField(offset, schema.getFieldType(i)); } } } else { @@ -275,7 +278,9 @@ protected Object parseFlatRecord(FlatRecordTraversalNode node) { for (int i = 0; i < flatRecordSchema.numFields(); i++) { int posInPojoSchema = schema.getPosition(flatRecordSchema.getFieldName(i)); if (posInPojoSchema != -1) { - mappedFields.get(posInPojoSchema).copy(obj, objectNode); + mappedFields.get(posInPojoSchema).copy(obj, reader, offset); + } else { + reader.skipField(offset, schema.getFieldType(i)); } } } @@ -766,48 +771,64 @@ private Object parseBoxedWrapper(GenericHollowObject record) { } } - private Object parseBoxedWrapper(FlatRecordTraversalObjectNode record) { + private Object parseBoxedWrapper(FlatRecordOrdinalReader reader, FlatRecordOrdinalReader.Offset offset) { switch (fieldType) { case BOOLEAN: - return record.getFieldValueBooleanBoxed(fieldName); + return reader.readFieldBoolean(offset); case INT: - return record.getFieldValueIntBoxed(fieldName); + int intValue = reader.readFieldInt(offset); + if (intValue == Integer.MIN_VALUE) { + return null; + } + return intValue; case SHORT: - int shortValue = record.getFieldValueInt(fieldName); + int shortValue = reader.readFieldInt(offset); if (shortValue == Integer.MIN_VALUE) { return null; } return Short.valueOf((short) shortValue); case BYTE: - int byteValue = record.getFieldValueInt(fieldName); + int byteValue = reader.readFieldInt(offset); if (byteValue == Integer.MIN_VALUE) { return null; } return Byte.valueOf((byte) byteValue); case CHAR: - int charValue = record.getFieldValueInt(fieldName); + int charValue = reader.readFieldInt(offset); if (charValue == Integer.MIN_VALUE) { return null; } return Character.valueOf((char) charValue); case LONG: - return record.getFieldValueLongBoxed(fieldName); + long longValue = reader.readFieldLong(offset); + if (longValue == Long.MIN_VALUE) { + return null; + } + return Long.valueOf(longValue); case FLOAT: - return record.getFieldValueFloatBoxed(fieldName); + float floatValue = reader.readFieldFloat(offset); + if (Float.isNaN(floatValue)) { + return null; + } + return floatValue; case DOUBLE: - return record.getFieldValueDoubleBoxed(fieldName); + double doubleValue = reader.readFieldDouble(offset); + if (Double.isNaN(doubleValue)) { + return null; + } + return doubleValue; case STRING: - return record.getFieldValueString(fieldName); + return reader.readFieldString(offset); case BYTES: - return record.getFieldValueBytes(fieldName); + return reader.readFieldBytes(offset); case ENUM_NAME: - String enumName = record.getFieldValueString(fieldName); + String enumName = reader.readFieldString(offset); if (enumName == null) { return null; } return Enum.valueOf((Class) clazz, enumName); case DATE_TIME: { - long dateValue = record.getFieldValueLong(fieldName); + long dateValue = reader.readFieldLong(offset); if (dateValue == Long.MIN_VALUE) { return null; } @@ -818,159 +839,159 @@ private Object parseBoxedWrapper(FlatRecordTraversalObjectNode record) { } } - private void copy(Object obj, FlatRecordTraversalObjectNode node) { + private void copy(Object obj, FlatRecordOrdinalReader reader, FlatRecordOrdinalReader.Offset offset) { switch (fieldType) { case BOOLEAN: { - Boolean value = node.getFieldValueBooleanBoxed(fieldName); + Boolean value = reader.readFieldBoolean(offset); if (value != null) { unsafe.putBoolean(obj, fieldOffset, value == Boolean.TRUE); } break; } case INT: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putInt(obj, fieldOffset, value); } break; } case SHORT: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putShort(obj, fieldOffset, (short) value); } break; } case BYTE: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putByte(obj, fieldOffset, (byte) value); } break; } case CHAR: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putChar(obj, fieldOffset, (char) value); } break; } case LONG: { - long value = node.getFieldValueLong(fieldName); + long value = reader.readFieldLong(offset); if (value != Long.MIN_VALUE) { unsafe.putLong(obj, fieldOffset, value); } break; } case FLOAT: { - float value = node.getFieldValueFloat(fieldName); + float value = reader.readFieldFloat(offset); if (!Float.isNaN(value)) { unsafe.putFloat(obj, fieldOffset, value); } break; } case DOUBLE: { - double value = node.getFieldValueDouble(fieldName); + double value = reader.readFieldDouble(offset); if (!Double.isNaN(value)) { unsafe.putDouble(obj, fieldOffset, value); } break; } case STRING: { - String value = node.getFieldValueString(fieldName); + String value = reader.readFieldString(offset); if (value != null) { unsafe.putObject(obj, fieldOffset, value); } break; } case BYTES: { - byte[] value = node.getFieldValueBytes(fieldName); + byte[] value = reader.readFieldBytes(offset); if (value != null) { unsafe.putObject(obj, fieldOffset, value); } break; } case INLINED_BOOLEAN: { - Boolean value = node.getFieldValueBooleanBoxed(fieldName); + Boolean value = reader.readFieldBoolean(offset); if (value != null) { unsafe.putObject(obj, fieldOffset, value); } break; } case INLINED_INT: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putObject(obj, fieldOffset, Integer.valueOf(value)); } break; } case INLINED_SHORT: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putObject(obj, fieldOffset, Short.valueOf((short) value)); } break; } case INLINED_BYTE: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putObject(obj, fieldOffset, Byte.valueOf((byte) value)); } break; } case INLINED_CHAR: { - int value = node.getFieldValueInt(fieldName); + int value = reader.readFieldInt(offset); if (value != Integer.MIN_VALUE) { unsafe.putObject(obj, fieldOffset, Character.valueOf((char) value)); } break; } case INLINED_LONG: { - long value = node.getFieldValueLong(fieldName); + long value = reader.readFieldLong(offset); if (value != Long.MIN_VALUE) { unsafe.putObject(obj, fieldOffset, Long.valueOf(value)); } break; } case INLINED_FLOAT: { - float value = node.getFieldValueFloat(fieldName); + float value = reader.readFieldFloat(offset); if (!Float.isNaN(value)) { unsafe.putObject(obj, fieldOffset, Float.valueOf(value)); } break; } case INLINED_DOUBLE: { - double value = node.getFieldValueDouble(fieldName); + double value = reader.readFieldDouble(offset); if (!Double.isNaN(value)) { unsafe.putObject(obj, fieldOffset, Double.valueOf(value)); } break; } case INLINED_STRING: { - String value = node.getFieldValueString(fieldName); + String value = reader.readFieldString(offset); if (value != null) { unsafe.putObject(obj, fieldOffset, value); } break; } case DATE_TIME: { - long value = node.getFieldValueLong(fieldName); + long value = reader.readFieldLong(offset); if (value != Long.MIN_VALUE) { unsafe.putObject(obj, fieldOffset, new Date(value)); } break; } case ENUM_NAME: { - String value = node.getFieldValueString(fieldName); + String value = reader.readFieldString(offset); if (value != null) { unsafe.putObject(obj, fieldOffset, Enum.valueOf((Class) type, value)); } break; } case REFERENCE: { - FlatRecordTraversalNode childNode = node.getFieldNode(fieldName); - if (childNode != null) { - unsafe.putObject(obj, fieldOffset, subTypeMapper.parseFlatRecord(childNode)); + int reference = reader.readFieldReference(offset); + if (reference != -1) { + unsafe.putObject(obj, fieldOffset, subTypeMapper.parseFlatRecord(reader, reference)); } break; } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java index c4896b302..9b5fef4b1 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java @@ -25,9 +25,8 @@ import com.netflix.hollow.core.write.HollowTypeWriteState; import com.netflix.hollow.core.write.HollowWriteRecord; import com.netflix.hollow.core.write.HollowWriteStateEngine; +import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordOrdinalReader; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalNode; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalSetNode; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -117,10 +116,14 @@ protected Object parseHollowRecord(HollowRecord record) { } @Override - protected Object parseFlatRecord(FlatRecordTraversalNode node) { + protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { + FlatRecordOrdinalReader.Offset offset = reader.getOffsetAtDataStartOf(ordinal); + int size = reader.readSize(offset); Set collection = new HashSet<>(); - for (FlatRecordTraversalNode elementNode : (FlatRecordTraversalSetNode) node) { - collection.add(elementMapper.parseFlatRecord(elementNode)); + int elementOrdinal = 0; + for (int i = 0; i < size; i++) { + elementOrdinal += reader.readSetElementOrdinalDelta(offset); + collection.add(elementMapper.parseFlatRecord(reader, elementOrdinal)); } return collection; } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowTypeMapper.java index 3d023c3fb..1ed8d5eab 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowTypeMapper.java @@ -21,8 +21,8 @@ import com.netflix.hollow.core.write.HollowTypeWriteState; import com.netflix.hollow.core.write.HollowWriteRecord; import com.netflix.hollow.core.write.HollowWriteStateEngine; +import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordOrdinalReader; import com.netflix.hollow.core.write.objectmapper.flatrecords.FlatRecordWriter; -import com.netflix.hollow.core.write.objectmapper.flatrecords.traversal.FlatRecordTraversalNode; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -46,8 +46,8 @@ public abstract class HollowTypeMapper { protected abstract Object parseHollowRecord(HollowRecord record); - protected abstract Object parseFlatRecord(FlatRecordTraversalNode node); - + protected abstract Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal); + protected abstract HollowWriteRecord newWriteRecord(); protected abstract HollowTypeWriteState getTypeWriteState(); diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java index f512f31e0..4900778d5 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java @@ -32,11 +32,27 @@ public int getOrdinalCount() { return ordinalOffsets.size(); } + public Offset getOffsetOf(int ordinal) { + return new Offset(getOrdinalOffset(ordinal)); + } + + public Offset getOffsetAtDataStartOf(int ordinal) { + int ordinalOffset = getOrdinalOffset(ordinal); + int schemaId = VarInt.readVInt(record.data, ordinalOffset); + return new Offset(ordinalOffset + VarInt.sizeOfVInt(schemaId)); + } + public HollowSchema readSchema(int ordinal) { int schemaId = VarInt.readVInt(record.data, getOrdinalOffset(ordinal)); return record.schemaIdMapper.getSchema(schemaId); } + public HollowSchema readSchema(Offset offset) { + int schemaId = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(schemaId)); + return record.schemaIdMapper.getSchema(schemaId); + } + public int readSize(int ordinal) { int offset = getOrdinalOffset(ordinal); @@ -53,6 +69,42 @@ public int readSize(int ordinal) { return VarInt.readVInt(record.data, offset); } + public int readSize(Offset offset) { + int schemaId = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(schemaId)); + + HollowSchema schema = record.schemaIdMapper.getSchema(schemaId); + if (schema.getSchemaType() != HollowSchema.SchemaType.LIST && + schema.getSchemaType() != HollowSchema.SchemaType.SET && + schema.getSchemaType() != HollowSchema.SchemaType.MAP) { + throw new IllegalArgumentException(String.format("Schema %s is not a LIST, SET, or MAP type (found %s)", schema.getName(), schema.getSchemaType())); + } + + int size = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(size)); + return size; + } + + public int readListElementOrdinal(Offset offset) { + int elementOrdinal = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(elementOrdinal)); + return elementOrdinal; + } + + public int readSetElementOrdinalDelta(Offset offset) { + int elementOrdinalDelta = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(elementOrdinalDelta)); + return elementOrdinalDelta; + } + + public long readMapKeyOrdinalDeltaAndValueOrdinal(Offset offset) { + int keyOrdinalDelta = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(keyOrdinalDelta)); + int valueOrdinal = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(valueOrdinal)); + return (long) keyOrdinalDelta << 32 | valueOrdinal; + } + public void readListElementsInto(int ordinal, int[] elements) { int offset = getOrdinalOffset(ordinal); @@ -126,67 +178,105 @@ public int readFieldReference(int ordinal, String field) { if (offset == -1) { return -1; } - if (VarInt.readVNull(record.data, offset)) { return -1; } - return VarInt.readVInt(record.data, offset); } + public int readFieldReference(Offset offset) { + if (VarInt.readVNull(record.data, offset.get())) { + offset.increment(1); + return -1; + } + int value = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(value)); + return value; + } + public Boolean readFieldBoolean(int ordinal, String field) { int offset = skipToField(ordinal, HollowObjectSchema.FieldType.BOOLEAN, field); if (offset == -1) { return null; } - if (VarInt.readVNull(record.data, offset)) { return null; } - int value = record.data.get(offset); return value == 1 ? Boolean.TRUE : Boolean.FALSE; } + public Boolean readFieldBoolean(Offset offset) { + offset.increment(1); + if (VarInt.readVNull(record.data, offset.get())) { + return null; + } + int value = record.data.get(offset.get()); + return value == 1 ? Boolean.TRUE : Boolean.FALSE; + } + public int readFieldInt(int ordinal, String field) { int offset = skipToField(ordinal, HollowObjectSchema.FieldType.INT, field); if (offset == -1) { return Integer.MIN_VALUE; } - if (VarInt.readVNull(record.data, offset)) { return Integer.MIN_VALUE; } - int value = VarInt.readVInt(record.data, offset); return ZigZag.decodeInt(value); } + public int readFieldInt(Offset offset) { + if (VarInt.readVNull(record.data, offset.get())) { + offset.increment(1); + return Integer.MIN_VALUE; + } + int value = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(value)); + return ZigZag.decodeInt(value); + } + public long readFieldLong(int ordinal, String field) { int offset = skipToField(ordinal, HollowObjectSchema.FieldType.LONG, field); if (offset == -1) { return Long.MIN_VALUE; } - if (VarInt.readVNull(record.data, offset)) { return Long.MIN_VALUE; } - long value = VarInt.readVLong(record.data, offset); return ZigZag.decodeLong(value); } + public long readFieldLong(Offset offset) { + if (VarInt.readVNull(record.data, offset.get())) { + offset.increment(1); + return Long.MIN_VALUE; + } + long value = VarInt.readVLong(record.data, offset.get()); + offset.increment(VarInt.sizeOfVLong(value)); + return ZigZag.decodeLong(value); + } + public float readFieldFloat(int ordinal, String field) { int offset = skipToField(ordinal, HollowObjectSchema.FieldType.FLOAT, field); if (offset == -1) { return Float.NaN; } - int value = record.data.readIntBits(offset); if (value == HollowObjectWriteRecord.NULL_FLOAT_BITS) { return Float.NaN; } + return Float.intBitsToFloat(value); + } + public float readFieldFloat(Offset offset) { + offset.increment(4); + int value = record.data.readIntBits(offset.get()); + if (value == HollowObjectWriteRecord.NULL_FLOAT_BITS) { + return Float.NaN; + } return Float.intBitsToFloat(value); } @@ -204,6 +294,15 @@ public double readFieldDouble(int ordinal, String field) { return Double.longBitsToDouble(value); } + public double readFieldDouble(Offset offset) { + offset.increment(8); + long value = record.data.readLongBits(offset.get()); + if (value == HollowObjectWriteRecord.NULL_DOUBLE_BITS) { + return Double.NaN; + } + return Double.longBitsToDouble(value); + } + public String readFieldString(int ordinal, String field) { int offset = skipToField(ordinal, HollowObjectSchema.FieldType.STRING, field); if (offset == -1) { @@ -228,6 +327,26 @@ public String readFieldString(int ordinal, String field) { return new String(s); } + public String readFieldString(Offset offset) { + if (VarInt.readVNull(record.data, offset.get())) { + offset.increment(1); + return null; + } + + int length = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(length)); + + int cLength = VarInt.countVarIntsInRange(record.data, offset.get(), length); + char[] s = new char[cLength]; + for (int i = 0; i < cLength; i++) { + int charValue = VarInt.readVInt(record.data, offset.get()); + s[i] = (char) charValue; + offset.increment(VarInt.sizeOfVInt(charValue)); + } + + return new String(s); + } + public byte[] readFieldBytes(int ordinal, String field) { int offset = skipToField(ordinal, HollowObjectSchema.FieldType.BYTES, field); if (offset == -1) { @@ -249,6 +368,29 @@ public byte[] readFieldBytes(int ordinal, String field) { return b; } + public byte[] readFieldBytes(Offset offset) { + if (VarInt.readVNull(record.data, offset.get())) { + offset.increment(1); + return null; + } + + int length = VarInt.readVInt(record.data, offset.get()); + offset.increment(VarInt.sizeOfVInt(length)); + + byte[] b = new byte[length]; + for (int i = 0; i < length; i++) { + b[i] = record.data.get(offset.get()); + offset.increment(1); + } + + return b; + } + + public void skipField(Offset offset, HollowObjectSchema.FieldType fieldType) { + int size = sizeOfFieldValue(fieldType, offset.get()); + offset.increment(size); + } + private int skipToField(int ordinal, HollowObjectSchema.FieldType fieldType, String field) { int offset = getOrdinalOffset(ordinal); @@ -339,4 +481,20 @@ private int sizeOfFieldValue(HollowObjectSchema.FieldType fieldType, int offset) throw new IllegalArgumentException("Unsupported field type: " + fieldType); } } + + public static class Offset { + public int offset; + + Offset(int offset) { + this.offset = offset; + } + + public void increment(int delta) { + offset += delta; + } + + public int get() { + return offset; + } + } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalListNode.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalListNode.java index 19ad9f44b..63893a735 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalListNode.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalListNode.java @@ -35,6 +35,11 @@ public int getOrdinal() { return ordinal; } + @Override + public FlatRecordOrdinalReader getReader() { + return reader; + } + @Override public void setCommonSchema(Map commonSchema) { this.commonSchemaMap = commonSchema; diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalMapNode.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalMapNode.java index 4e9454771..67d6eb773 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalMapNode.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalMapNode.java @@ -40,6 +40,11 @@ public int getOrdinal() { return ordinal; } + @Override + public FlatRecordOrdinalReader getReader() { + return reader; + } + @Override public void setCommonSchema(Map commonSchema) { this.commonSchemaMap = commonSchema; diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalNode.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalNode.java index 550d561a9..4cf5fa952 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalNode.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalNode.java @@ -15,6 +15,8 @@ public interface FlatRecordTraversalNode { int getOrdinal(); + FlatRecordOrdinalReader getReader(); + HollowSchema getSchema(); void setCommonSchema(Map commonSchema); diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalObjectNode.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalObjectNode.java index 1a303289b..46bea22f5 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalObjectNode.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalObjectNode.java @@ -37,6 +37,11 @@ public int getOrdinal() { return ordinal; } + @Override + public FlatRecordOrdinalReader getReader() { + return reader; + } + @Override public void setCommonSchema(Map commonSchema) { this.commonSchemaMap = commonSchema; diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalSetNode.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalSetNode.java index d14cb8496..4c5ca116f 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalSetNode.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/traversal/FlatRecordTraversalSetNode.java @@ -36,6 +36,11 @@ public int getOrdinal() { return ordinal; } + @Override + public FlatRecordOrdinalReader getReader() { + return reader; + } + @Override public void setCommonSchema(Map commonSchema) { this.commonSchemaMap = commonSchema; From 13bf4428cefb9a4a068f5ebaa838cb79c786541c Mon Sep 17 00:00:00 2001 From: Eduardo Ramirez Date: Fri, 25 Oct 2024 14:31:44 -0700 Subject: [PATCH 2/3] fix --- .../objectmapper/flatrecords/FlatRecordOrdinalReader.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java index 4900778d5..16fe54682 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java @@ -207,11 +207,12 @@ public Boolean readFieldBoolean(int ordinal, String field) { } public Boolean readFieldBoolean(Offset offset) { - offset.increment(1); if (VarInt.readVNull(record.data, offset.get())) { + offset.increment(1); return null; } int value = record.data.get(offset.get()); + offset.increment(1); return value == 1 ? Boolean.TRUE : Boolean.FALSE; } @@ -272,8 +273,8 @@ public float readFieldFloat(int ordinal, String field) { } public float readFieldFloat(Offset offset) { - offset.increment(4); int value = record.data.readIntBits(offset.get()); + offset.increment(4); if (value == HollowObjectWriteRecord.NULL_FLOAT_BITS) { return Float.NaN; } @@ -295,8 +296,8 @@ public double readFieldDouble(int ordinal, String field) { } public double readFieldDouble(Offset offset) { - offset.increment(8); long value = record.data.readLongBits(offset.get()); + offset.increment(8); if (value == HollowObjectWriteRecord.NULL_DOUBLE_BITS) { return Double.NaN; } From 6f8c350caaade8fbeb288b7da5b7328a53d46d72 Mon Sep 17 00:00:00 2001 From: Eduardo Ramirez Date: Fri, 25 Oct 2024 15:52:49 -0700 Subject: [PATCH 3/3] fixes --- .../objectmapper/HollowMapTypeMapper.java | 10 +++++----- .../objectmapper/HollowObjectTypeMapper.java | 6 +++--- .../objectmapper/HollowSetTypeMapper.java | 6 +++--- .../flatrecords/FlatRecordOrdinalReader.java | 18 ++++-------------- ...apperFlatRecordTraversalNodeParserTest.java | 1 + 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java index 26bb2a38b..578adb233 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowMapTypeMapper.java @@ -143,13 +143,13 @@ protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { FlatRecordOrdinalReader.Offset offset = reader.getOffsetAtDataStartOf(ordinal); int size = reader.readSize(offset); Map collection = new HashMap<>(size); - int keyOrdinal = 0; + int previousKeyOrdinal = 0; for (int i = 0; i < size; i++) { - long keyOrdinalDeltaAndValueOrdinal = reader.readMapKeyOrdinalDeltaAndValueOrdinal(offset); - keyOrdinal += (int) (keyOrdinalDeltaAndValueOrdinal >>> 32); - int valueOrdinal = (int) keyOrdinalDeltaAndValueOrdinal; + long keyAndValueOrdinals = reader.readMapKeyAndValueOrdinals(offset, previousKeyOrdinal); + previousKeyOrdinal = (int) (keyAndValueOrdinals >>> 32); + int valueOrdinal = (int) keyAndValueOrdinals; - Object key = keyMapper.parseFlatRecord(reader, keyOrdinal); + Object key = keyMapper.parseFlatRecord(reader, previousKeyOrdinal); Object value = valueMapper.parseFlatRecord(reader, valueOrdinal); collection.put(key, value); } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java index 34935d29d..633c6d916 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowObjectTypeMapper.java @@ -257,7 +257,7 @@ protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { if (posInPojoSchema != -1) { obj = mappedFields.get(posInPojoSchema).parseBoxedWrapper(reader, offset); } else { - reader.skipField(offset, schema.getFieldType(i)); + reader.skipField(offset, flatRecordSchema.getFieldType(i)); } } } else if (clazz.isEnum()) { @@ -270,7 +270,7 @@ protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { if (fieldName.equals(MappedFieldType.ENUM_NAME.getSpecialFieldName()) && posInPojoSchema != -1) { obj = mappedFields.get(posInPojoSchema).parseBoxedWrapper(reader, offset); } else { - reader.skipField(offset, schema.getFieldType(i)); + reader.skipField(offset, flatRecordSchema.getFieldType(i)); } } } else { @@ -280,7 +280,7 @@ protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { if (posInPojoSchema != -1) { mappedFields.get(posInPojoSchema).copy(obj, reader, offset); } else { - reader.skipField(offset, schema.getFieldType(i)); + reader.skipField(offset, flatRecordSchema.getFieldType(i)); } } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java index 9b5fef4b1..a8760b26e 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/HollowSetTypeMapper.java @@ -120,10 +120,10 @@ protected Object parseFlatRecord(FlatRecordOrdinalReader reader, int ordinal) { FlatRecordOrdinalReader.Offset offset = reader.getOffsetAtDataStartOf(ordinal); int size = reader.readSize(offset); Set collection = new HashSet<>(); - int elementOrdinal = 0; + int previousElementOrdinal = 0; for (int i = 0; i < size; i++) { - elementOrdinal += reader.readSetElementOrdinalDelta(offset); - collection.add(elementMapper.parseFlatRecord(reader, elementOrdinal)); + previousElementOrdinal = reader.readSetElementOrdinal(offset, previousElementOrdinal); + collection.add(elementMapper.parseFlatRecord(reader, previousElementOrdinal)); } return collection; } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java index 16fe54682..a801c7c1b 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/objectmapper/flatrecords/FlatRecordOrdinalReader.java @@ -70,16 +70,6 @@ public int readSize(int ordinal) { } public int readSize(Offset offset) { - int schemaId = VarInt.readVInt(record.data, offset.get()); - offset.increment(VarInt.sizeOfVInt(schemaId)); - - HollowSchema schema = record.schemaIdMapper.getSchema(schemaId); - if (schema.getSchemaType() != HollowSchema.SchemaType.LIST && - schema.getSchemaType() != HollowSchema.SchemaType.SET && - schema.getSchemaType() != HollowSchema.SchemaType.MAP) { - throw new IllegalArgumentException(String.format("Schema %s is not a LIST, SET, or MAP type (found %s)", schema.getName(), schema.getSchemaType())); - } - int size = VarInt.readVInt(record.data, offset.get()); offset.increment(VarInt.sizeOfVInt(size)); return size; @@ -91,18 +81,18 @@ public int readListElementOrdinal(Offset offset) { return elementOrdinal; } - public int readSetElementOrdinalDelta(Offset offset) { + public int readSetElementOrdinal(Offset offset, int previousElementOrdinal) { int elementOrdinalDelta = VarInt.readVInt(record.data, offset.get()); offset.increment(VarInt.sizeOfVInt(elementOrdinalDelta)); - return elementOrdinalDelta; + return previousElementOrdinal + elementOrdinalDelta; } - public long readMapKeyOrdinalDeltaAndValueOrdinal(Offset offset) { + public long readMapKeyAndValueOrdinals(Offset offset, int previousKeyOrdinal) { int keyOrdinalDelta = VarInt.readVInt(record.data, offset.get()); offset.increment(VarInt.sizeOfVInt(keyOrdinalDelta)); int valueOrdinal = VarInt.readVInt(record.data, offset.get()); offset.increment(VarInt.sizeOfVInt(valueOrdinal)); - return (long) keyOrdinalDelta << 32 | valueOrdinal; + return (long) (previousKeyOrdinal + keyOrdinalDelta) << 32 | valueOrdinal; } public void readListElementsInto(int ordinal, int[] elements) { diff --git a/hollow/src/test/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapperFlatRecordTraversalNodeParserTest.java b/hollow/src/test/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapperFlatRecordTraversalNodeParserTest.java index caa4f8e99..d7fa4be30 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapperFlatRecordTraversalNodeParserTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/write/objectmapper/HollowObjectMapperFlatRecordTraversalNodeParserTest.java @@ -19,6 +19,7 @@ import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; + public class HollowObjectMapperFlatRecordTraversalNodeParserTest { private HollowObjectMapper mapper; private FlatRecordWriter flatRecordWriter;