Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Lock the Polypheny Home Directory #501

Merged
merged 56 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from 55 commits
Commits
Show all changes
56 commits
Select commit Hold shift + click to select a range
c879e47
Fix comments
gartens Jun 18, 2024
22db948
Annotate the entire class with Getter
gartens Jun 18, 2024
64aa224
Use toList
gartens Jun 24, 2024
7997b8d
Improve error message
gartens Jun 24, 2024
6c17799
Use advisory locks to ensure only one Polypheny instance is using a d…
gartens Apr 23, 2024
f6bfc8e
Remove check that is no longer necessary
gartens Apr 23, 2024
dbebec8
Remove unused variable and minor formatting
gartens Apr 23, 2024
d5d3ece
Add comment to apparently unused variable
gartens Apr 23, 2024
f91aa91
Remove unused import
gartens Apr 23, 2024
a01a29f
Mark generateOrLoadPolyphenyUUID as static
gartens Apr 23, 2024
01b1b8f
Stop Polypheny when resetting the catalog fails
gartens Apr 23, 2024
ddc7cda
Move helper class out of runPolyphenyDb
gartens Apr 23, 2024
6839574
Use correct variable in probeCreatingFolder
gartens Apr 23, 2024
3945d50
Remove leftover duration measurement
gartens Jun 14, 2024
ae90b90
Use .toList() instead of .collect( Collectors.toList() )
gartens Jun 24, 2024
815121b
Use Long.compare to prevent over/underflows
gartens Jun 11, 2024
80bec83
Fix whitespace
gartens Jun 10, 2024
50a1015
Prevent overflow/underflow by using Long.compare
gartens Jun 25, 2024
06399b2
Remove unused namespaceId from IdBuilder
gartens Jun 25, 2024
de23229
Handle plain EOFException in handleMessages
gartens Jun 25, 2024
00b890c
Stricter visibility for fields in InMemoryPersister and FilePersister
gartens Jun 25, 2024
206d9fa
Remove unused methods
gartens Jun 25, 2024
a76cc6f
Use GenericRuntimeExceptions
gartens Jun 25, 2024
9f9969f
Format
gartens Jun 25, 2024
f1d960c
Remove unused methods
gartens Jun 25, 2024
5a83f24
Remove todo
gartens Jun 25, 2024
b91af79
Fix typo if -> id
gartens Jun 26, 2024
2bf11bf
Remove annotations implied by @Value
gartens Jun 27, 2024
4c0ab2c
Remove implied annotations
gartens Jun 27, 2024
ada767d
Use consistent method names
gartens Jun 27, 2024
3c33af5
Remove unused generics
gartens Jun 27, 2024
3e23d5a
Remove unused method
gartens Jun 28, 2024
d046fa6
Make the name match the UI
gartens Jun 28, 2024
962b021
Rename interfaceName to interfaceType
gartens Jun 28, 2024
0bb902c
Remove unused token from MQL grammar
gartens Jun 25, 2024
03b1cf4
Remove adapter catalogs when dropping an adapter
gartens Jul 2, 2024
1219ca3
Make related classes look similar
gartens Jul 3, 2024
867c9ee
Replace unwrap().orElseThrow() with unwrapOrElse()
gartens Jul 3, 2024
432dd3d
Replace .collect( Collectors.toList() ) with .toList()
gartens Jul 3, 2024
42da6c7
Improve readability by casting earlier
gartens Jul 3, 2024
688e6f9
Reduce code duplication
gartens Jun 27, 2024
facd088
Make small improvements to DDLManagerImpl
gartens Jul 15, 2024
305af82
Use correct method in createPrimaryKey
gartens Jul 16, 2024
903d4f4
Use class name as error message when t.getMessage() is null
gartens Jul 16, 2024
2b78798
Prevent null-pointer dereference when closing the statement
gartens Jul 16, 2024
dcc0bef
Minor code and formatting improvements
vogti Jul 28, 2024
1470147
Fix field names
gartens Jul 29, 2024
20f50fe
Replace for loop with forEach
gartens Jul 3, 2024
824f23e
Replace for loops with .count
gartens Jul 30, 2024
2436fad
Replace containsKey/put/get with computeIfAbsent, putIfAbsent
gartens Jul 30, 2024
5211a9e
Simplify some enums
gartens Jul 30, 2024
a6133f3
Replace some for loops with streams
gartens Jul 30, 2024
c8f817c
Replace List with Set
gartens Jul 30, 2024
369a78f
added some missing reformatting
datomo Aug 5, 2024
82dcf88
mini cleanup in MongoFilter of unused code
datomo Aug 5, 2024
ecefed3
Fix entire awt import
vogti Aug 5, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,10 @@

import java.io.File;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
Expand All @@ -39,10 +42,11 @@ public class PolyphenyHomeDirManager {

private static PolyphenyHomeDirManager INSTANCE = null;

private File root;
private final File root;
private File home;
private final List<File> dirs = new ArrayList<>();
private final List<File> deleteOnExit = new ArrayList<>();
private final FileLock lock; // Reference so the lock is not released
@Getter
private static RunMode mode;

Expand Down Expand Up @@ -73,6 +77,18 @@ private PolyphenyHomeDirManager() {
}
}

File lockFile = registerNewFile( ".lock" );
try {
FileChannel fileChannel = FileChannel.open( lockFile.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE );
Optional<FileLock> fileLock = Optional.ofNullable( fileChannel.tryLock() );
if ( fileLock.isPresent() ) {
lock = fileLock.get();
} else {
throw new RuntimeException( "There is already another Polypheny instance running in this directory" );
}
} catch ( IOException e ) {
throw new RuntimeException( "Failed to open lockfile", e );
}
Runtime.getRuntime().addShutdownHook( new Thread( () -> {
for ( File file : deleteOnExit ) {
if ( file.exists() ) {
Expand Down Expand Up @@ -103,16 +119,16 @@ private String getPrefix() {
}


private boolean probeCreatingFolder( File file ) {
private static boolean probeCreatingFolder( File file ) {
if ( file.isFile() ) {
return false;
}

boolean couldCreate = true;
if ( !home.exists() ) {
couldCreate = home.mkdirs();
if ( !file.exists() ) {
couldCreate = file.mkdirs();
}
return couldCreate && home.canWrite();
return couldCreate && file.canWrite();
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ public static JsonSerializer<DataSource<?>> getSerializer() {
jsonSource.addProperty( "uniqueName", src.getUniqueName() );
jsonSource.addProperty( "adapterName", src.getAdapterName() );
jsonSource.add( "adapterSettings", context.serialize( AbstractAdapterSetting.serializeSettings( src.getAvailableSettings( src.getClass() ), src.getCurrentSettings() ) ) );
jsonSource.add( "currentSettings", context.serialize( src.getCurrentSettings() ) );
jsonSource.add( "settings", context.serialize( src.getCurrentSettings() ) );
jsonSource.add( "dataReadOnly", context.serialize( src.isDataReadOnly() ) );
jsonSource.addProperty( "type", src.getAdapterType().name() );
return jsonSource;
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/polypheny/db/adapter/DataStore.java
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public static JsonSerializer<DataStore<?>> getSerializer() {
JsonObject jsonStore = new JsonObject();
jsonStore.addProperty( "adapterId", src.getAdapterId() );
jsonStore.add( "adapterSettings", context.serialize( AbstractAdapterSetting.serializeSettings( src.getAvailableSettings( src.getClass() ), src.getCurrentSettings() ) ) );
jsonStore.add( "currentSettings", context.serialize( src.getCurrentSettings() ) );
jsonStore.add( "settings", context.serialize( src.getCurrentSettings() ) );
jsonStore.addProperty( "adapterName", src.getAdapterName() );
jsonStore.addProperty( "uniqueName", src.getUniqueName() );
jsonStore.addProperty( "type", src.getAdapterType().name() );
Expand Down
3 changes: 1 addition & 2 deletions core/src/main/java/org/polypheny/db/adapter/index/Index.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import lombok.Getter;
import org.polypheny.db.PolyImplementation;
import org.polypheny.db.ResultIterator;
Expand Down Expand Up @@ -97,7 +96,7 @@ public void rebuild( final Transaction transaction ) {
}
final AlgNode scan = builder
.relScan( table )
.project( cols.stream().map( builder::field ).collect( Collectors.toList() ) )
.project( cols.stream().map( builder::field ).toList() )
.build();
final QueryProcessor processor = statement.getQueryProcessor();
final PolyImplementation implementation = processor.prepareQuery( AlgRoot.of( scan, Kind.SELECT ), false );
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import org.polypheny.db.adapter.DataStore.IndexMethodModel;
import org.polypheny.db.adapter.index.Index.IndexFactory;
import org.polypheny.db.catalog.Catalog;
Expand Down Expand Up @@ -223,7 +222,7 @@ public Index getIndex( LogicalNamespace schema, LogicalTable table, List<String>
public List<Index> getIndices( LogicalNamespace schema, LogicalTable table ) {
return this.indexById.values().stream()
.filter( index -> index.schema.equals( schema ) && index.table.equals( table ) )
.collect( Collectors.toList() );
.toList();
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ public RexNode asSingleProject() {
// null key is replaceRoot
nodes.add(
builder.makeLiteral(
PolyList.copyOf( includes.keySet().stream().filter( Objects::nonNull ).map( v -> PolyList.copyOf( Arrays.stream( v.split( "\\." ) ).map( PolyString::of ).collect( Collectors.toList() ) ) )
.collect( Collectors.toList() ) ),
PolyList.copyOf( includes.keySet().stream().filter( Objects::nonNull ).map( v -> PolyList.copyOf( Arrays.stream( v.split( "\\." ) ).map( PolyString::of ).toList() ) )
.toList() ),
builder.getTypeFactory().createArrayType( builder.getTypeFactory().createPolyType( PolyType.CHAR, 255 ), -1 ), PolyType.ARRAY ) );
nodes.addAll( includes.entrySet().stream().filter( o -> Objects.nonNull( o.getKey() ) ).map( Entry::getValue ).toList() );

Expand All @@ -115,7 +115,7 @@ public RexNode asSingleProject() {
builder.getTypeFactory().createArrayType(
builder.getTypeFactory().createArrayType( builder.getTypeFactory().createPolyType( PolyType.CHAR, 255 ), -1 ),
-1 ),
excludes.stream().map( value -> PolyList.of( Arrays.stream( value.split( "\\." ) ).map( PolyString::of ).collect( Collectors.toList() ) ) ).collect( Collectors.toList() ) ) );
excludes.stream().map( value -> PolyList.of( Arrays.stream( value.split( "\\." ) ).map( PolyString::of ).toList() ) ).collect( Collectors.toList() ) ) );
}

return doc;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Getter;
import org.polypheny.db.algebra.AlgNode;
import org.polypheny.db.algebra.SingleAlg;
Expand Down Expand Up @@ -85,12 +84,12 @@ public NodeType getNodeType() {

@Override
public AlgNode accept( RexShuttle shuttle ) {
List<RexNode> exp = this.matches.stream().map( m -> (RexNode) m ).collect( Collectors.toList() );
List<RexNode> exp = this.matches.stream().map( m -> (RexNode) m ).toList();
List<RexNode> exps = shuttle.apply( exp );
if ( exp == exps ) {
return this;
}
return new LogicalLpgMatch( getCluster(), traitSet, input, exps.stream().map( e -> (RexCall) e ).collect( Collectors.toList() ), names );
return new LogicalLpgMatch( getCluster(), traitSet, input, exps.stream().map( e -> (RexCall) e ).toList(), names );
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@

import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import lombok.Getter;
import org.polypheny.db.algebra.AlgNode;
import org.polypheny.db.algebra.SingleAlg;
Expand Down Expand Up @@ -89,7 +88,7 @@ protected AlgDataType deriveRowType() {

@Override
public AlgNode accept( RexShuttle shuttle ) {
List<RexNode> exp = this.projects.stream().map( p -> (RexNode) p ).collect( Collectors.toList() );
List<RexNode> exp = this.projects.stream().map( p -> (RexNode) p ).toList();
List<RexNode> exps = shuttle.apply( exp );
if ( exp == exps ) {
return this;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import lombok.NonNull;
import org.polypheny.db.algebra.AlgInput;
Expand Down Expand Up @@ -92,7 +91,7 @@ public AlgDataType deriveRowType() {
* Returns an identity projection for the given table.
*/
public static ImmutableList<Integer> identity( Entity entity ) {
return ImmutableList.copyOf( IntStream.range( 0, entity.getTupleType().getFieldCount() ).boxed().collect( Collectors.toList() ) );
return ImmutableList.copyOf( IntStream.range( 0, entity.getTupleType().getFieldCount() ).boxed().toList() );
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.Getter;
import org.apache.calcite.linq4j.Enumerable;
Expand Down Expand Up @@ -140,7 +139,7 @@ public ClassDeclaration implementRoot( EnumerableAlg rootAlg, EnumerableAlg.Pref
if ( contextCounter < 0 ) {
// If the context was not set by ContextSwitcher we only need the initial one
Statement assign = Expressions.declare( Modifier.FINAL, DataContext.ROOT, DataContext.INITIAL_ROOT );
block = Expressions.block( Stream.concat( Stream.of( assign ), block.statements.stream() ).collect( Collectors.toList() ) );
block = Expressions.block( Stream.concat( Stream.of( assign ), block.statements.stream() ).toList() );
}

// add values
Expand Down Expand Up @@ -516,4 +515,3 @@ public void go( EnumerableAlg.Result result ) {
}

}

Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import java.lang.reflect.Modifier;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.calcite.linq4j.Enumerable;
import org.apache.calcite.linq4j.function.Function;
import org.apache.calcite.linq4j.function.Function0;
Expand Down Expand Up @@ -91,7 +90,7 @@ public Result implement( EnumerableAlgImplementor implementor, Prefer pref ) {
Expressions.constant( DataContext.ROOT ),
builder.append( builder.newName( "query" + System.nanoTime() ), query.block() ),
exp,
Expressions.constant( getLeft().getTupleType().getFields().stream().map( f -> f.getType().getPolyType() ).collect( Collectors.toList() ) ) );
Expressions.constant( getLeft().getTupleType().getFields().stream().map( f -> f.getType().getPolyType() ).toList() ) );

builder.add( Expressions.return_( null, builder.append( "test", transformContext ) ) );

Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/polypheny/db/catalog/Catalog.java
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public static void afterInit( Runnable action ) {

public abstract AllocationGraphCatalog getAllocGraph( long namespaceId );

public abstract <S extends AdapterCatalog> Optional<S> getAdapterCatalog( long id );
public abstract Optional<AdapterCatalog> getAdapterCatalog( long id );

public abstract void addStoreSnapshot( AdapterCatalog snapshot );

Expand Down
12 changes: 0 additions & 12 deletions core/src/main/java/org/polypheny/db/catalog/IdBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,6 @@ public class IdBuilder {
@Serialize
public AtomicLong snapshotId;

@Serialize
public AtomicLong namespaceId;

@Serialize
public AtomicLong entityId;

Expand Down Expand Up @@ -99,14 +96,12 @@ private IdBuilder() {
new AtomicLong( 0 ),
new AtomicLong( 0 ),
new AtomicLong( 0 ),
new AtomicLong( 0 ),
new AtomicLong( 0 ) );
}


public IdBuilder(
@Deserialize("snapshotId") AtomicLong snapshotId,
@Deserialize("namespaceId") AtomicLong namespaceId,
@Deserialize("entityId") AtomicLong entityId,
@Deserialize("fieldId") AtomicLong fieldId,
@Deserialize("userId") AtomicLong userId,
Expand All @@ -122,7 +117,6 @@ public IdBuilder(
@Deserialize("partitionId") AtomicLong partitionId,
@Deserialize("placementId") AtomicLong placementId ) {
this.snapshotId = snapshotId;
this.namespaceId = namespaceId;
this.entityId = entityId;
this.fieldId = fieldId;

Expand Down Expand Up @@ -157,11 +151,6 @@ public long getNewFieldId() {
}


public long getNewNamespaceId() {
return namespaceId.getAndIncrement();
}


public long getNewUserId() {
return userId.getAndIncrement();
}
Expand Down Expand Up @@ -224,7 +213,6 @@ public long getNewAdapterTemplateId() {

public synchronized void restore( IdBuilder idBuilder ) {
this.snapshotId.set( idBuilder.snapshotId.longValue() );
this.namespaceId.set( idBuilder.namespaceId.longValue() );
this.entityId.set( idBuilder.entityId.longValue() );
this.fieldId.set( idBuilder.fieldId.longValue() );

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
import lombok.experimental.NonFinal;
import lombok.extern.slf4j.Slf4j;
import org.apache.calcite.linq4j.tree.Expression;
import org.apache.calcite.linq4j.tree.Expressions;
import org.polypheny.db.catalog.Catalog;
import org.polypheny.db.catalog.entity.allocation.AllocationEntity;
import org.polypheny.db.catalog.entity.physical.PhysicalEntity;
Expand Down Expand Up @@ -90,7 +89,7 @@ public AdapterCatalog(


public Expression asExpression() {
return Expressions.call( Catalog.CATALOG_EXPRESSION, "getAdapterCatalog", Expressions.constant( adapterId ) );
return Catalog.PHYSICAL_EXPRESSION.apply( adapterId );
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
import java.util.stream.Collectors;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.Value;
import org.polypheny.db.catalog.IdBuilder;
import org.polypheny.db.catalog.entity.allocation.AllocationCollection;
Expand All @@ -41,7 +39,6 @@
import org.polypheny.db.catalog.entity.physical.PhysicalTable;
import org.polypheny.db.util.Pair;

@Getter
@EqualsAndHashCode(callSuper = true)
@Value
public class DocAdapterCatalog extends AdapterCatalog {
Expand All @@ -57,7 +54,7 @@ public void renameLogicalColumn( long id, String newFieldName ) {
List<PhysicalColumn> updates = new ArrayList<>();
for ( PhysicalField field : fields.values() ) {
if ( field.id == id ) {
updates.add( field.unwrap( PhysicalColumn.class ).orElseThrow().toBuilder().logicalName( newFieldName ).build() );
updates.add( field.unwrapOrThrow( PhysicalColumn.class ).toBuilder().logicalName( newFieldName ).build() );
}
}
for ( PhysicalColumn u : updates ) {
Expand All @@ -77,7 +74,7 @@ public DocAdapterCatalog(


public <T extends PhysicalEntity> T fromAllocation( long id, Class<T> clazz ) {
return getPhysicalsFromAllocs( id ).get( 0 ).unwrap( clazz ).orElseThrow();
return getPhysicalsFromAllocs( id ).get( 0 ).unwrapOrThrow( clazz );
}


Expand All @@ -98,7 +95,7 @@ public void addColumn( PhysicalColumn column ) {


public PhysicalColumn getColumn( long columnId, long allocId ) {
return fields.get( Pair.of( allocId, columnId ) ).unwrap( PhysicalColumn.class ).orElseThrow();
return fields.get( Pair.of( allocId, columnId ) ).unwrapOrThrow( PhysicalColumn.class );
}


Expand All @@ -119,6 +116,7 @@ public PhysicalColumn updateColumnType( long allocId, LogicalColumn newCol ) {
return column;
}


public PhysicalTable createTable(
String namespaceName,
String tableName,
Expand All @@ -129,7 +127,7 @@ public PhysicalTable createTable(
AllocationTableWrapper wrapper ) {
AllocationTable allocation = wrapper.table;
List<AllocationColumn> columns = wrapper.columns;
List<PhysicalColumn> pColumns = columns.stream().map( c -> new PhysicalColumn( columnNames.get( c.columnId ), logical.id, allocation.id, allocation.adapterId, c.position, logicalColumns.get( c.columnId ) ) ).collect( Collectors.toList() );
List<PhysicalColumn> pColumns = columns.stream().map( c -> new PhysicalColumn( columnNames.get( c.columnId ), logical.id, allocation.id, allocation.adapterId, c.position, logicalColumns.get( c.columnId ) ) ).toList();
long physicalId = IdBuilder.getInstance().getNewPhysicalId();
PhysicalTable table = new PhysicalTable( physicalId, allocation.id, allocation.logicalId, tableName, pColumns, logical.namespaceId, namespaceName, pkIds, allocation.adapterId );
pColumns.forEach( this::addColumn );
Expand Down
Loading