Skip to content

Use RDF4J native store in place of Blazegraph #583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
balhoff wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Use RDF4J native store in place of Blazegraph #583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions minerva-cli/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,20 @@
<dependency>
<groupId>org.obolibrary.robot</groupId>
<artifactId>robot-core</artifactId>
<exclusions>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>log4j-over-slf4j</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
Expand Down
107 changes: 35 additions & 72 deletions minerva-cli/src/main/java/org/geneontology/minerva/cli/CommandLineInterface.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
package org.geneontology.minerva.cli;


import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSailRepository;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
import com.google.common.base.Optional;
import com.google.common.collect.Sets;
import com.google.gson.Gson;
Expand All @@ -14,6 +10,10 @@
import org.apache.commons.lang3.StringUtils;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
import org.eclipse.rdf4j.sail.NotifyingSail;
import org.eclipse.rdf4j.sail.nativerdf.NativeStore;
import org.geneontology.minerva.*;
import org.geneontology.minerva.curie.CurieHandler;
import org.geneontology.minerva.curie.CurieMappings;
Expand All @@ -31,23 +31,22 @@
import org.geneontology.minerva.server.handler.OperationsTools;
import org.geneontology.minerva.server.inferences.InferenceProviderCreator;
import org.geneontology.minerva.server.validation.MinervaShexValidator;
import org.geneontology.minerva.util.BlazegraphMutationCounter;
import org.geneontology.minerva.util.SailMutationCounter;
import org.geneontology.minerva.validation.ShexValidationReport;
import org.geneontology.minerva.validation.ValidationResultSet;
import org.geneontology.minerva.validation.Violation;
import org.geneontology.minerva.validation.pipeline.BatchPipelineValidationReport;
import org.geneontology.minerva.validation.pipeline.ErrorMessage;
import org.obolibrary.robot.CatalogXmlIRIMapper;
import org.openrdf.model.Statement;
import org.openrdf.model.ValueFactory;
import org.openrdf.model.impl.URIImpl;
import org.openrdf.model.vocabulary.OWL;
import org.openrdf.model.vocabulary.RDF;
import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.QueryLanguage;
import org.openrdf.query.UpdateExecutionException;
import org.openrdf.repository.RepositoryException;
import org.openrdf.rio.*;
import org.eclipse.rdf4j.model.Statement;
import org.eclipse.rdf4j.model.ValueFactory;
import org.eclipse.rdf4j.model.vocabulary.OWL;
import org.eclipse.rdf4j.model.vocabulary.RDF;
import org.eclipse.rdf4j.query.MalformedQueryException;
import org.eclipse.rdf4j.query.QueryLanguage;
import org.eclipse.rdf4j.query.UpdateExecutionException;
import org.eclipse.rdf4j.repository.RepositoryException;
import org.eclipse.rdf4j.rio.*;
import org.semanticweb.owlapi.apibinding.OWLManager;
import org.semanticweb.owlapi.formats.TurtleDocumentFormat;
import org.semanticweb.owlapi.io.IRIDocumentSource;
Expand Down Expand Up @@ -553,62 +552,31 @@ public static void modelsToJSON(String ontologyFileURL, String journalFilePath,
public static void importOWLModels(String journalFilePath, String inputFolder) throws Exception {
// minimal inputs
if (journalFilePath == null) {
System.err.println("No journal file was configured.");
System.exit(-1);
LOGGER.fatal("No journal file was configured.");
System.exit(1);
return;
}
if (inputFolder == null) {
System.err.println("No input folder was configured.");
System.exit(-1);
LOGGER.fatal("No input folder was configured.");
System.exit(1);
return;
}
File repositoryDir = new File(journalFilePath);
if (repositoryDir.exists() && !repositoryDir.isDirectory()) {
LOGGER.fatal("Repository not a directory");
System.exit(1);
return;
}
int total_files = 0;
if (repositoryDir.exists() && repositoryDir.listFiles().length > 0) {
LOGGER.warn("Repository already exists; note that bulk load does not check for previously loaded model IRIs and content may be merged.");
}
OWLOntology dummy = OWLManager.createOWLOntologyManager().createOntology(IRI.create("http://example.org/dummy"));
String modelIdPrefix = "http://model.geneontology.org/"; // this will not be used for anything
CurieHandler curieHandler = new MappedCurieHandler();
BlazegraphMolecularModelManager<Void> m3 = new BlazegraphMolecularModelManager<>(dummy, curieHandler, modelIdPrefix, journalFilePath, null, null, false);
//in case of update rather than whole new journal
Set<IRI> stored = new HashSet<IRI>(m3.getStoredModelIds());
LOGGER.info("loading gocams from " + inputFolder);
//for (File file : FileUtils.listFiles(new File(inputFolder), null, true)) {
File i = new File(inputFolder);
if (i.exists()) {
if (i.isDirectory()) {
total_files = i.listFiles().length;
FileUtils.listFiles(i, null, true).parallelStream().parallel().forEach(file -> {
if (file.getName().endsWith("ttl")) {
java.util.Optional<String> irio;
try {
irio = m3.scanForOntologyIRI(file);
IRI iri = null;
if (irio.isPresent()) {
iri = IRI.create(irio.get());
}
//is it in there already?
if (stored.contains(iri)) {
LOGGER.error("Attempted to load gocam ttl file into database but gocam with that iri already exists, skipping " + file + " " + iri);
} else {
stored.add(iri);
m3.importModelToDatabase(file, true);
}
} catch (RDFParseException | RDFHandlerException | IOException e1) {
// TODO Auto-generated catch block
e1.printStackTrace();
} catch (OWLOntologyCreationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (RepositoryException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else {
LOGGER.info("Ignored for not ending with .ttl" + file);
}
});
}
}
m3.importBulkModelsToDatabase(i, true);
m3.dispose();
LOGGER.info("done loading gocams, loaded: " + stored.size() + " out of: " + total_files + " files");
}

/**
Expand Down Expand Up @@ -719,21 +687,16 @@ public static void sparqlUpdate(String journalFilePath, String updateFile) throw
System.exit(-1);
return;
}

String update = FileUtils.readFileToString(new File(updateFile), StandardCharsets.UTF_8);
Properties properties = new Properties();
properties.load(CommandLineInterface.class.getResourceAsStream("/org/geneontology/minerva/blazegraph.properties"));
properties.setProperty(com.bigdata.journal.Options.FILE, journalFilePath);

BigdataSail sail = new BigdataSail(properties);
BigdataSailRepository repository = new BigdataSailRepository(sail);
repository.initialize();
BigdataSailRepositoryConnection conn = repository.getUnisolatedConnection();
BlazegraphMutationCounter counter = new BlazegraphMutationCounter();
conn.addChangeLog(counter);
String indexes = "spoc,posc,cosp"; //FIXME review for appropriate indexes
NotifyingSail sail = new NativeStore(new File(journalFilePath), indexes);
SailRepository repository = new SailRepository(sail);
SailRepositoryConnection conn = repository.getConnection();
SailMutationCounter counter = new SailMutationCounter();
sail.addSailChangedListener(counter);
conn.prepareUpdate(QueryLanguage.SPARQL, update).execute();
int changes = counter.mutationCount();
conn.removeChangeLog(counter);
sail.removeSailChangedListener(counter);
System.out.println("\nApplied " + changes + " changes");
conn.close();
}
Expand Down
28 changes: 11 additions & 17 deletions minerva-cli/src/main/java/org/geneontology/minerva/cli/ReplaceObsoleteReferencesCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,21 @@
package org.geneontology.minerva.cli;

import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSailRepository;
import org.apache.log4j.Logger;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.sail.nativerdf.NativeStore;
import org.geneontology.minerva.MolecularModelManager;
import org.geneontology.minerva.curie.CurieHandler;
import org.geneontology.minerva.curie.DefaultCurieHandler;
import org.geneontology.minerva.util.BlazegraphMutationCounter;
import org.geneontology.minerva.util.SailMutationCounter;
import org.obolibrary.robot.CatalogXmlIRIMapper;
import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.UpdateExecutionException;
import org.openrdf.repository.RepositoryException;
import org.eclipse.rdf4j.query.MalformedQueryException;
import org.eclipse.rdf4j.query.UpdateExecutionException;
import org.eclipse.rdf4j.repository.RepositoryException;
import org.semanticweb.owlapi.apibinding.OWLManager;
import org.semanticweb.owlapi.model.*;
import org.semanticweb.owlapi.model.parameters.Imports;

import java.io.File;
import java.io.IOException;
import java.util.Optional;
import java.util.Properties;
Expand Down Expand Up @@ -58,21 +59,14 @@ public static void run(String ontologyIRI, String catalogPath, String journalFil
} catch (OWLOntologyCreationException e) {
throw new FatalReplaceObsoleteReferencesError("Could not load tbox ontology from " + ontologyIRI, e);
}
Properties properties = new Properties();
String indexes = "spoc,posc,cosp"; //FIXME review for appropriate indexes
SailRepository repository;
try {
properties.load(CommandLineInterface.class.getResourceAsStream("/org/geneontology/minerva/blazegraph.properties"));
} catch (IOException e) {
throw new FatalReplaceObsoleteReferencesError("Could not read blazegraph properties resource from jar file.");
}
properties.setProperty(com.bigdata.journal.Options.FILE, journalFilePath);
BigdataSail sail = new BigdataSail(properties);
BigdataSailRepository repository = new BigdataSailRepository(sail);
try {
repository.initialize();
repository = new SailRepository(new NativeStore(new File(journalFilePath), indexes));
} catch (RepositoryException e) {
throw new FatalReplaceObsoleteReferencesError("Could not initialize SAIL repository for database.", e);
}
BlazegraphMutationCounter counter = new BlazegraphMutationCounter();
SailMutationCounter counter = new SailMutationCounter();
String replacements = createReplacementsValuesList(tbox);
String sparqlUpdate = classReplacementUpdateTemplate.replace("%%%values%%%", replacements);
String complementsSparqlUpdate = complementsUpdateTemplate.replace("%%%values%%%", replacements);
Expand Down
72 changes: 40 additions & 32 deletions minerva-cli/src/main/java/org/geneontology/minerva/cli/ReplaceTermsCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,26 @@
package org.geneontology.minerva.cli;

import com.bigdata.rdf.changesets.IChangeLog;
import com.bigdata.rdf.sail.BigdataSail;
import com.bigdata.rdf.sail.BigdataSailRepository;
import com.bigdata.rdf.sail.BigdataSailRepositoryConnection;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.log4j.Logger;
import org.eclipse.rdf4j.repository.Repository;
import org.eclipse.rdf4j.repository.sail.SailRepository;
import org.eclipse.rdf4j.repository.sail.SailRepositoryConnection;
import org.eclipse.rdf4j.sail.NotifyingSail;
import org.eclipse.rdf4j.sail.Sail;
import org.eclipse.rdf4j.sail.nativerdf.NativeStore;
import org.geneontology.minerva.MolecularModelManager;
import org.geneontology.minerva.curie.CurieHandler;
import org.geneontology.minerva.curie.DefaultCurieHandler;
import org.geneontology.minerva.util.BlazegraphMutationCounter;
import org.obolibrary.robot.CatalogXmlIRIMapper;
import org.openrdf.query.MalformedQueryException;
import org.openrdf.query.QueryLanguage;
import org.openrdf.query.UpdateExecutionException;
import org.openrdf.repository.RepositoryException;
import org.semanticweb.owlapi.apibinding.OWLManager;
import org.geneontology.minerva.util.SailMutationCounter;
import org.eclipse.rdf4j.query.MalformedQueryException;
import org.eclipse.rdf4j.query.QueryLanguage;
import org.eclipse.rdf4j.query.UpdateExecutionException;
import org.eclipse.rdf4j.repository.RepositoryException;
import org.semanticweb.owlapi.model.IRI;
import org.semanticweb.owlapi.model.OWLAnnotationValue;
import org.semanticweb.owlapi.model.OWLLiteral;
import org.semanticweb.owlapi.model.OWLOntologyManager;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
Expand Down Expand Up @@ -72,21 +70,14 @@ public static void run(String journalFilePath, String replacementClassesPath, St
if (journalFilePath == null) {
throw new FatalTermReplacementError("No journal file was configured.");
}
Properties properties = new Properties();
String indexes = "spoc,posc,cosp"; //FIXME review for appropriate indexes
SailRepository repository;
try {
properties.load(CommandLineInterface.class.getResourceAsStream("/org/geneontology/minerva/blazegraph.properties"));
} catch (IOException e) {
throw new FatalTermReplacementError("Could not read blazegraph properties resource from jar file.");
}
properties.setProperty(com.bigdata.journal.Options.FILE, journalFilePath);
BigdataSail sail = new BigdataSail(properties);
BigdataSailRepository repository = new BigdataSailRepository(sail);
try {
repository.initialize();
repository = new SailRepository(new NativeStore(new File(journalFilePath), indexes));
} catch (RepositoryException e) {
throw new FatalTermReplacementError("Could not initialize SAIL repository for database.", e);
}
BlazegraphMutationCounter counter = new BlazegraphMutationCounter();
SailMutationCounter counter = new SailMutationCounter();
String classReplacements = formatAsSPARQLValuesList(loadTermReplacementFromFile(replacementClassesPath));
String objectPropertyReplacements = formatAsSPARQLValuesList(loadTermReplacementFromFile(replacementPropertiesPath));
String classesSparqlUpdate = classReplacementUpdateTemplate.replace("%%%values%%%", classReplacements);
Expand Down Expand Up @@ -130,21 +121,38 @@ private static String formatAsSPARQLValuesList(Set<Pair<Pair<IRI, String>, Pair<
.collect(Collectors.joining(" "));
}

protected static void applySPARQLUpdate(BigdataSailRepository repository, String update, Optional<IChangeLog> changeLog) throws RepositoryException, UpdateExecutionException, MalformedQueryException {
BigdataSailRepositoryConnection connection = repository.getUnisolatedConnection();
changeLog.ifPresent(connection::addChangeLog);
try {
protected static void applySPARQLUpdate(SailRepository repository, String update, Optional<SailMutationCounter> counter) throws RepositoryException, UpdateExecutionException, MalformedQueryException {
try (SailRepositoryConnection connection = repository.getConnection()) {
final Repository repo = connection.getRepository();
NotifyingSail notifyingSail;
if (repo instanceof SailRepository) {
Sail sail = ((SailRepository) repo).getSail();
if (sail instanceof NotifyingSail) {
notifyingSail = (NotifyingSail) sail;
} else {
notifyingSail = null;
}
} else {
notifyingSail = null;
}
connection.begin();
counter.ifPresent(c -> {
if (notifyingSail != null) {
notifyingSail.addSailChangedListener(c);
}
});
try {
connection.prepareUpdate(QueryLanguage.SPARQL, update).execute();
} catch (UpdateExecutionException | RepositoryException | MalformedQueryException e) {
connection.rollback();
throw e;
}
} finally {
connection.close();
counter.ifPresent(c -> {
if (notifyingSail != null) {
notifyingSail.removeSailChangedListener(c);
}
});
}
changeLog.ifPresent(connection::removeChangeLog);
}

private static Optional<IRI> curieToIRI(String curie) {
Expand Down
13 changes: 9 additions & 4 deletions minerva-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,6 @@
<artifactId>minerva-json</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.blazegraph</groupId>
<artifactId>bigdata-core</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
Expand Down Expand Up @@ -92,6 +88,15 @@
<artifactId>commons-math3</artifactId>
<version>3.6.1</version>
</dependency>
<dependency>
<groupId>org.eclipse.rdf4j</groupId>
<artifactId>rdf4j-storage</artifactId>
<type>pom</type>
</dependency>
<dependency>
<groupId>org.eclipse.rdf4j</groupId>
<artifactId>rdf4j-sail-nativerdf</artifactId>
</dependency>
<dependency>
<groupId>org.obolibrary.robot</groupId>
<artifactId>robot-core</artifactId>
Expand Down
Loading
Loading