Skip to content

Commit

Permalink
AVRO-3473: ServiceLoader for Conversion (#1624)
Browse files Browse the repository at this point in the history
* AVRO-3473: ServiceLoader for Conversion

Automatically register Conversion classes using the Java 6 service
loader upon startup. This works for any Conversion that does not need
constructor parameters.

* AVRO-3473: ServiceLoader for Conversion

Improved variable name.

* AVRO-3473: Spotless

* Apply suggestions from code review

Co-authored-by: Ryan Skraba <[email protected]>

* AVRO-3473: Removed newline to trigger build

* AVRO-3473: Add missing import

No idea when that got dropped...

* AVRO-3473: Removed unused import

* AVRO-3473: Fix rebase errors

* Fix static imports

---------

Co-authored-by: Ryan Skraba <[email protected]>
  • Loading branch information
opwvhk and RyanSkraba committed Jun 15, 2023
1 parent adc0b5a commit 72beda3
Show file tree
Hide file tree
Showing 10 changed files with 238 additions and 49 deletions.
43 changes: 28 additions & 15 deletions lang/java/avro/src/main/java/org/apache/avro/Conversion.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,30 +21,43 @@
import java.nio.ByteBuffer;
import java.util.Collection;
import java.util.Map;
import java.util.ServiceLoader;

import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericEnumSymbol;
import org.apache.avro.generic.GenericFixed;
import org.apache.avro.generic.IndexedRecord;

/**
* Conversion between generic and logical type instances.
* <p>
* Instances of this class are added to GenericData to convert a logical type to
* a particular representation.
* Instances of this class can be added to GenericData to convert a logical type
* to a particular representation. This can be done manually, using
* {@link GenericData#addLogicalTypeConversion(Conversion)}, or automatically.
* This last option uses the Java {@link ServiceLoader}, and requires the
* implementation to be a public class with a public no-arg constructor, be
* named in a file called {@code /META-INF/services/org.apache.avro.Conversion},
* and both must available in the classpath.</li>
* <p>
* Implementations must provide: * {@link #getConvertedType()}: get the Java
* class used for the logical type * {@link #getLogicalTypeName()}: get the
* logical type this implements
* Implementations must provide:
* <ul>
* <li>{@link #getConvertedType()}: get the Java class used for the logical
* type</li>
* <li>{@link #getLogicalTypeName()}: get the logical type this implements</li>
* </ul>
* <p>
* Subclasses must also override all of the conversion methods for Avro's base
* types that are valid for the logical type, or else risk causing
* Subclasses must also override the conversion methods for Avro's base types
* that are valid for the logical type, or else risk causing
* {@code UnsupportedOperationException} at runtime.
* <p>
* Optionally, use {@link #getRecommendedSchema()} to provide a Schema that will
* be used when a Schema is generated for the class returned by
* {@code getConvertedType}.
* be used when generating a Schema for the class. This is useful when using
* {@code ReflectData} or {@code ProtobufData}, for example.
*
* @param <T> a Java type that generic data is converted to
* @param <T> a Java type that can represent the named logical type
* @see ServiceLoader
*/
@SuppressWarnings("unused")
public abstract class Conversion<T> {

/**
Expand All @@ -65,9 +78,9 @@ public abstract class Conversion<T> {
* Certain logical types may require adjusting the code within the "setter"
* methods to make sure the data that is set is properly formatted. This method
* allows the Conversion to generate custom setter code if required.
*
* @param varName
* @param valParamName
*
* @param varName the name of the variable holding the converted value
* @param valParamName the name of the parameter with the new converted value
* @return a String for the body of the setter method
*/
public String adjustAndSetValue(String varName, String valParamName) {
Expand Down Expand Up @@ -102,7 +115,7 @@ public T fromCharSequence(CharSequence value, Schema schema, LogicalType type) {
throw new UnsupportedOperationException("fromCharSequence is not supported for " + type.getName());
}

public T fromEnumSymbol(GenericEnumSymbol value, Schema schema, LogicalType type) {
public T fromEnumSymbol(GenericEnumSymbol<?> value, Schema schema, LogicalType type) {
throw new UnsupportedOperationException("fromEnumSymbol is not supported for " + type.getName());
}

Expand Down Expand Up @@ -150,7 +163,7 @@ public CharSequence toCharSequence(T value, Schema schema, LogicalType type) {
throw new UnsupportedOperationException("toCharSequence is not supported for " + type.getName());
}

public GenericEnumSymbol toEnumSymbol(T value, Schema schema, LogicalType type) {
public GenericEnumSymbol<?> toEnumSymbol(T value, Schema schema, LogicalType type) {
throw new UnsupportedOperationException("toEnumSymbol is not supported for " + type.getName());
}

Expand Down
29 changes: 15 additions & 14 deletions lang/java/avro/src/main/java/org/apache/avro/Conversions.java
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,12 @@ public GenericFixed toFixed(BigDecimal value, Schema schema, LogicalType type) {
byte fillByte = (byte) (value.signum() < 0 ? 0xFF : 0x00);
byte[] unscaled = value.unscaledValue().toByteArray();
byte[] bytes = new byte[schema.getFixedSize()];
int offset = bytes.length - unscaled.length;
int unscaledLength = unscaled.length;
int offset = bytes.length - unscaledLength;

// Fill the front of the array and copy remaining with unscaled values
// Fill the front with the filler and copy the unscaled value into the remainder
Arrays.fill(bytes, 0, offset, fillByte);
System.arraycopy(unscaled, 0, bytes, offset, bytes.length - offset);
System.arraycopy(unscaled, 0, bytes, offset, unscaledLength);

return new GenericData.Fixed(schema, bytes);
}
Expand Down Expand Up @@ -147,7 +148,7 @@ private static BigDecimal validate(final LogicalTypes.Decimal decimal, BigDecima
}

/**
* Convert a underlying representation of a logical type (such as a ByteBuffer)
* Convert an underlying representation of a logical type (such as a ByteBuffer)
* to a higher level object (such as a BigDecimal).
*
* @param datum The object to be converted.
Expand All @@ -157,9 +158,9 @@ private static BigDecimal validate(final LogicalTypes.Decimal decimal, BigDecima
* @param conversion The tool used to finish the conversion. Cannot be null if
* datum is not null.
* @return The result object, which is a high level object of the logical type.
* If a null datum is passed in, a null value will be returned.
* @throws IllegalArgumentException if a null schema, type or conversion is
* passed in while datum is not null.
* The null datum always converts to a null value.
* @throws IllegalArgumentException if datum is not null, but schema, type or
* conversion is.
*/
public static Object convertToLogicalType(Object datum, Schema schema, LogicalType type, Conversion<?> conversion) {
if (datum == null) {
Expand All @@ -176,9 +177,9 @@ public static Object convertToLogicalType(Object datum, Schema schema, LogicalTy
case RECORD:
return conversion.fromRecord((IndexedRecord) datum, schema, type);
case ENUM:
return conversion.fromEnumSymbol((GenericEnumSymbol) datum, schema, type);
return conversion.fromEnumSymbol((GenericEnumSymbol<?>) datum, schema, type);
case ARRAY:
return conversion.fromArray((Collection) datum, schema, type);
return conversion.fromArray((Collection<?>) datum, schema, type);
case MAP:
return conversion.fromMap((Map<?, ?>) datum, schema, type);
case FIXED:
Expand All @@ -201,13 +202,13 @@ public static Object convertToLogicalType(Object datum, Schema schema, LogicalTy
return datum;
} catch (ClassCastException e) {
throw new AvroRuntimeException(
"Cannot convert " + datum + ":" + datum.getClass().getSimpleName() + ": expected generic type", e);
"Cannot convert " + datum + ':' + datum.getClass().getSimpleName() + ": expected generic type", e);
}
}

/**
* Convert a high level representation of a logical type (such as a BigDecimal)
* to the its underlying representation object (such as a ByteBuffer)
* to its underlying representation object (such as a ByteBuffer)
*
* @param datum The object to be converted.
* @param schema The schema of datum. Cannot be null if datum is not null.
Expand All @@ -218,8 +219,8 @@ public static Object convertToLogicalType(Object datum, Schema schema, LogicalTy
* @return The result object, which is an underlying representation object of
* the logical type. If the input param datum is null, a null value will
* be returned.
* @throws IllegalArgumentException if a null schema, type or conversion is
* passed in while datum is not null.
* @throws IllegalArgumentException if datum is not null, but schema, type or
* conversion is.
*/
public static <T> Object convertToRawType(Object datum, Schema schema, LogicalType type, Conversion<T> conversion) {
if (datum == null) {
Expand Down Expand Up @@ -262,7 +263,7 @@ public static <T> Object convertToRawType(Object datum, Schema schema, LogicalTy
return datum;
} catch (ClassCastException e) {
throw new AvroRuntimeException(
"Cannot convert " + datum + ":" + datum.getClass().getSimpleName() + ": expected logical type", e);
"Cannot convert " + datum + ':' + datum.getClass().getSimpleName() + ": expected logical type", e);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.UUID;
import java.util.concurrent.ConcurrentMap;

Expand Down Expand Up @@ -117,13 +118,25 @@ public GenericData() {
/** For subclasses. GenericData does not use a ClassLoader. */
public GenericData(ClassLoader classLoader) {
this.classLoader = (classLoader != null) ? classLoader : getClass().getClassLoader();
loadConversions();
}

/** Return the class loader that's used (by subclasses). */
public ClassLoader getClassLoader() {
return classLoader;
}

/**
* Use the Java 6 ServiceLoader to load conversions.
*
* @see #addLogicalTypeConversion(Conversion)
*/
private void loadConversions() {
for (Conversion<?> conversion : ServiceLoader.load(Conversion.class, classLoader)) {
addLogicalTypeConversion(conversion);
}
}

private Map<String, Conversion<?>> conversions = new HashMap<>();

private Map<Class<?>, Map<String, Conversion<?>>> conversionsByClass = new IdentityHashMap<>();
Expand All @@ -134,19 +147,17 @@ public Collection<Conversion<?>> getConversions() {

/**
* Registers the given conversion to be used when reading and writing with this
* data model.
* data model. Conversions can also be registered automatically, as documented
* on the class {@link Conversion Conversion&lt;T&gt;}.
*
* @param conversion a logical type Conversion.
*/
public void addLogicalTypeConversion(Conversion<?> conversion) {
conversions.put(conversion.getLogicalTypeName(), conversion);
Class<?> type = conversion.getConvertedType();
Map<String, Conversion<?>> conversions = conversionsByClass.get(type);
if (conversions == null) {
conversions = new LinkedHashMap<>();
conversionsByClass.put(type, conversions);
}
conversions.put(conversion.getLogicalTypeName(), conversion);
Map<String, Conversion<?>> conversionsForClass = conversionsByClass.computeIfAbsent(type,
k -> new LinkedHashMap<>());
conversionsForClass.put(conversion.getLogicalTypeName(), conversion);
}

/**
Expand Down Expand Up @@ -187,11 +198,11 @@ public <T> Conversion<T> getConversionByClass(Class<T> datumClass, LogicalType l
* @return the conversion for the logical type, or null
*/
@SuppressWarnings("unchecked")
public Conversion<Object> getConversionFor(LogicalType logicalType) {
public <T> Conversion<T> getConversionFor(LogicalType logicalType) {
if (logicalType == null) {
return null;
}
return (Conversion<Object>) conversions.get(logicalType.getName());
return (Conversion<T>) conversions.get(logicalType.getName());
}

public static final String FAST_READER_PROP = "org.apache.avro.fastread";
Expand Down
48 changes: 48 additions & 0 deletions lang/java/avro/src/test/java/org/apache/avro/CustomType.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.avro;

import java.util.Objects;

public final class CustomType {
private final String name;

public CustomType(CharSequence name) {
this.name = name.toString();
}

public String getName() {
return name;
}

@Override
public int hashCode() {
return Objects.hashCode(name);
}

@Override
public boolean equals(Object obj) {
return obj instanceof CustomType && name.equals(((CustomType) obj).name);
}

@Override
public String toString() {
return "CustomType{name='" + name + "'}";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.avro;

public class CustomTypeConverter extends Conversion<CustomType> {
private static final CustomTypeLogicalTypeFactory logicalTypeFactory = new CustomTypeLogicalTypeFactory();

@Override
public Class<CustomType> getConvertedType() {
return CustomType.class;
}

@Override
public String getLogicalTypeName() {
return logicalTypeFactory.getTypeName();
}

@Override
public Schema getRecommendedSchema() {
return Schema.create(Schema.Type.STRING);
}

@Override
public CustomType fromCharSequence(CharSequence value, Schema schema, LogicalType type) {
return new CustomType(value);
}

@Override
public CharSequence toCharSequence(CustomType value, Schema schema, LogicalType type) {
return value.getName();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@
*/
package org.apache.avro;

public class DummyLogicalTypeFactory implements LogicalTypes.LogicalTypeFactory {
public class CustomTypeLogicalTypeFactory implements LogicalTypes.LogicalTypeFactory {
@Override
public LogicalType fromSchema(Schema schema) {
return LogicalTypes.date();
return new LogicalType(getTypeName());
}

@Override
public String getTypeName() {
return "service-example";
return "custom";
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -288,9 +288,9 @@ void registerLogicalTypeWithFactoryNameNotProvided() {
}

@Test
void registerLogicalTypeFactoryByServiceLoader() {
public void testRegisterLogicalTypeFactoryByServiceLoader() {
assertThat(LogicalTypes.getCustomRegisteredTypes(),
IsMapContaining.hasEntry(equalTo("service-example"), instanceOf(LogicalTypes.LogicalTypeFactory.class)));
IsMapContaining.hasEntry(equalTo("custom"), instanceOf(LogicalTypes.LogicalTypeFactory.class)));
}

public static void assertEqualsTrue(String message, Object o1, Object o2) {
Expand Down
Loading

0 comments on commit 72beda3

Please sign in to comment.