彈簧數據Neo4j超出了GC開銷限制

問題：我試圖用Spring應用程序將DBLP數據庫（從公開可用的XML file）導入到Neo4j實例中，但在應用程序將約39000個出版物導入到Neo4j數據庫我得到了「超出GC開銷限制」的消息。彈簧數據Neo4j超出了GC開銷限制

注意：我已徹底尋找答案在stackoverflow.com，但我一直沒能解決問題，即使這裏有很多很好的答案。

這裏是入口點Application.java：

package hello; 

import java.io.File; 

import org.neo4j.graphdb.GraphDatabaseService; 
import org.neo4j.graphdb.Transaction; 
import org.neo4j.graphdb.factory.GraphDatabaseFactory; 
import org.neo4j.kernel.impl.util.FileUtils; 
import org.springframework.beans.factory.annotation.Autowired; 
import org.springframework.boot.CommandLineRunner; 
import org.springframework.boot.SpringApplication; 
import org.springframework.boot.autoconfigure.SpringBootApplication; 
import org.springframework.context.annotation.Bean; 
import org.springframework.context.annotation.Configuration; 
import org.springframework.data.neo4j.config.EnableNeo4jRepositories; 
import org.springframework.data.neo4j.config.Neo4jConfiguration; 
import org.springframework.data.neo4j.core.GraphDatabase; 

import java.io.FileInputStream; 
import java.io.IOException; 
import java.io.InputStream; 

import javax.xml.parsers.ParserConfigurationException; 
import javax.xml.parsers.SAXParser; 
import javax.xml.parsers.SAXParserFactory; 

import org.xml.sax.SAXException; 
import org.apache.xerces.util.SecurityManager; 

/** 
* 
* Inspiration: http://spring.io/guides/gs/accessing-data-neo4j/ 
* 
*/ 
@SpringBootApplication 
public class Application implements CommandLineRunner { 

    @Configuration 
    @EnableNeo4jRepositories(basePackages = "hello") 
    static class ApplicationConfig extends Neo4jConfiguration { 

     public ApplicationConfig() { 
      setBasePackage("hello"); 
     } 

     @Bean 
     GraphDatabaseService graphDatabaseService() { 
      return new GraphDatabaseFactory().newEmbeddedDatabase("accessingdataneo4j.db"); 
     } 

    } 

    @Autowired 
    PublicationRepository publicationRepository; 

    @Autowired 
    GraphDatabase graphDatabase; 

    public void run(String... args) throws Exception { 

     Transaction tx = graphDatabase.beginTx(); 

     try { 

      SAXParserFactory parserFactory = SAXParserFactory.newInstance(); 
      SAXParser parser = parserFactory.newSAXParser(); 

      SecurityManager mgr = new SecurityManager(); 
      mgr.setEntityExpansionLimit(3100000); 
      parser.setProperty("http://apache.org/xml/properties/security-manager", mgr); 

      SaxHandler handler = new SaxHandler(publicationRepository); 
      parser.getXMLReader().setFeature("http://xml.org/sax/features/validation", true); 
      InputStream xmlInput = new FileInputStream("/Users/username/Downloads/dblp.xml"); 
      parser.parse(xmlInput, handler); 

      tx.success(); 

     } catch (SAXException e) { 
      e.printStackTrace(); 
     } catch (IOException e) { 
      e.printStackTrace(); 
     } catch (ParserConfigurationException e) { 
      e.printStackTrace(); 
     } finally { 
      tx.close(); 
     } 

    } 

    public static void main(String[] args) throws Exception { 
     FileUtils.deleteRecursively(new File("accessingdataneo4j.db")); 
     SpringApplication.run(Application.class, args); 
    } 

}

這裏是SAX處理程序Application.java利用的：

package hello; 

import java.util.Stack; 

import org.springframework.beans.factory.annotation.Autowired; 
import org.xml.sax.Attributes; 
import org.xml.sax.SAXException; 
import org.xml.sax.helpers.DefaultHandler; 

public class SaxHandler extends DefaultHandler { 

    private Stack<String> elementStack = new Stack<String>(); 
    private Stack<Publication> objectStack = new Stack<Publication>(); 
    private String publicationType = null; 
    private PublicationRepository publicationRepository = null; 
    private Publication publication = null; 
    private Author author = null; 
    private String currentElement = null; 
    private String value = null; 

    private static int counter = 0; 

    @Autowired 
    public SaxHandler(PublicationRepository publicationRepository) { 
     this.publicationRepository = publicationRepository; 
    } 

    public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { 

     elementStack.push(qName); 

     publication = new Publication(); 

     if ("article".equals(qName)) { 
      publication.setType("article"); 
     } else if ("inproceedings".equals(qName)) { 
      publication.setType("inproceedings"); 
     } else if ("proceedings".equals(qName)) { 
      publication.setType("proceedings"); 
     } else if ("book".equals(qName)) { 
      publication.setType("book"); 
     } else if ("incollection".equals(qName)) { 
      publication.setType("incollection"); 
     } else if ("phdthesis".equals(qName)) { 
      publication.setType("phdthesis"); 
     } else if ("mastersthesis".equals(qName)) { 
      publication.setType("mastersthesis"); 
     } else if ("www".equals(qName)) { 
      publication.setType("www"); 
     } 

     if (attributes.getLength() > 0) {   
      publicationType = qName; 

      if (attributes.getValue("key") != null) { 
       publication.setKey(attributes.getValue("key")); 
      } 
      if (attributes.getValue("mdate") != null) { 
       publication.setMdate(attributes.getValue("mdate")); 
      } 
      if (attributes.getValue("publtype") != null) { 
       publication.setMdate(attributes.getValue("publtype")); 
      } 
      if (attributes.getValue("reviewid") != null) { 
       publication.setMdate(attributes.getValue("reviewid")); 
      } 
      if (attributes.getValue("rating") != null) { 
       publication.setMdate(attributes.getValue("rating")); 
      } 

      objectStack.push(publication); 
     } 
    } 

    public void endElement(String uri, String localName, String qName) throws SAXException { 

     elementStack.pop(); 

     if (publicationType.equals(qName)) { 
      publicationRepository.save(objectStack.pop()); 
      ++counter; 
      if (counter % 1000 == 0) { 
       System.out.println("counter = " + counter); 
       System.out.println("element stack size = " + elementStack.size()); 
       System.out.println("object stack size = " + objectStack.size()); 
       for (Publication p : objectStack) { 
        System.out.println("objectStack: " + p); 
       } 
      } 
      if (counter % 5000 == 0) { 
       System.gc(); 
      } 
     } 

    } 

    public void characters(char ch[], int start, int length) throws SAXException { 

     value = new String(ch,start,length).trim(); 

     if (value.length() == 0) 
      return; 

     publication = objectStack.peek(); 
     currentElement = elementStack.peek(); 

     if ("author".equals(currentElement)) {   
      author = new Author(); 
      author.setName(value); 
      publication.addAuthor(author); 
     } else if ("editor".equals(currentElement)) { 
      publication.setEditor(value); 
     } else if ("title".equals(currentElement)) { 
      publication.setTitle(value); 
     } else if ("booktitle".equals(currentElement)) { 
      publication.setBooktitle(value); 
     } else if ("pages".equals(currentElement)) { 
      publication.setPages(value); 
     } else if ("year".equals(currentElement)) { 
      publication.setYear(value); 
     } else if ("address".equals(currentElement)) { 
      publication.setAddress(value); 
     } else if ("journal".equals(currentElement)) { 
      publication.setJournal(value); 
     } else if ("volume".equals(currentElement)) { 
      publication.setVolume(value); 
     } else if ("number".equals(currentElement)) { 
      publication.setNumber(value); 
     } else if ("month".equals(currentElement)) { 
      publication.setMonth(value); 
     } else if ("url".equals(currentElement)) { 
      publication.setUrl(value); 
     } else if ("ee".equals(currentElement)) { 
      publication.setEe(value); 
     } else if ("cdrom".equals(currentElement)) { 
      publication.setCdrom(value); 
     } else if ("cite".equals(currentElement)) { 
      publication.setCite(value); 
     } else if ("publisher".equals(currentElement)) { 
      publication.setPublisher(value); 
     } else if ("note".equals(currentElement)) { 
      publication.setNote(value); 
     } else if ("crossref".equals(currentElement)) { 
      publication.setCrossref(value); 
     } else if ("isbn".equals(currentElement)) { 
      publication.setIsbn(value); 
     } else if ("series".equals(currentElement)) { 
      publication.setSeries(value); 
     } else if ("school".equals(currentElement)) { 
      publication.setSchool(value); 
     } else if ("chapter".equals(currentElement)) { 
      publication.setChapter(value); 
     } 
    } 
}

這裏是PublicationRepository.java：

package hello; 

import org.springframework.data.repository.CrudRepository; 

public interface PublicationRepository extends CrudRepository<Publication, String> { 

    Publication findByTitle(String title); 

}

這是作者.java，其中一個域模型：

package hello; 

import org.springframework.data.neo4j.annotation.GraphId; 
import org.springframework.data.neo4j.annotation.NodeEntity; 

@NodeEntity 
public class Author { 

    @GraphId 
    private Long id; 

    private String name; 

    public Author() { 

    } 

    public Author(String name) { 

    } 

    @Override 
    public boolean equals(Object obj) { 
     if (this == obj) 
      return true; 

     if (obj == null) 
      return false; 

     if (this.getClass() != obj.getClass()) 
      return false; 

     Author other = (Author) obj; 

     if (this.id != null && this.name != null && other.id != null && other.name != null) { 
      if (this.id.equals(other.id) && this.name.equals(other.name)) 
       return true; 
     } else { 
      return true; 
     } 

     return false; 
    } 

    @Override 
    public int hashCode() { 
     return 31 * (this.id == null ? 1 : this.id.hashCode()) + 31 * (this.name == null ? 1 : this.name.hashCode()); 
    } 

    public Long getId() { 
     return id; 
    } 

    public void setId(Long id) { 
     this.id = id; 
    } 

    public String getName() { 
     return name; 
    } 

    public void setName(String name) { 
     this.name = name; 
    } 
}

這裏是Publication.java，第二域模型：

package hello; 

import java.io.Serializable; 
import java.util.HashSet; 
import java.util.Set; 

import org.neo4j.graphdb.Direction; 
import org.springframework.data.neo4j.annotation.GraphId; 
import org.springframework.data.neo4j.annotation.NodeEntity; 
import org.springframework.data.neo4j.annotation.RelatedTo; 

@NodeEntity 
public class Publication implements Serializable { 

    private static final long serialVersionUID = -6393545300391560520L; 

    @GraphId 
    Long nodeId; 

    private String type = ""; 
    private String key = ""; 
    private String mdate = ""; 
    private String publtype = ""; 
    private String reviewid = ""; 
    private String rating = ""; 

    @RelatedTo(type = "WROTE", direction = Direction.INCOMING) 
    private Set<Author> authors = new HashSet<Author>(); 
    private String editor = ""; 
    private String title = ""; 
    private String booktitle = ""; 
    private String pages = ""; 
    private String year = ""; 
    private String address = ""; 
    private String journal = ""; 
    private String volume = ""; 
    private String number = ""; 
    private String month = ""; 
    private String url = ""; 
    private String ee = ""; 
    private String cdrom = ""; 
    private String cite = ""; 
    private String publisher = ""; 
    private String note = ""; 
    private String crossref = ""; 
    private String isbn = ""; 
    private String series = ""; 
    private String school = ""; 
    private String chapter = ""; 

    public Publication() { 

    } 

    public void addAuthor(Author author) { 
     authors.add(author); 
    } 

    public Set<Author> getAuthors() { 
     return authors; 
    } 

    public void setAuthors(Set<Author> authors) { 
     this.authors = authors; 
    } 

    @Override 
    public String toString() { 
     return "TYPE: " + type + "\n" 
       + "KEY: " + key + "\n" 
       + "MDATE: " + mdate + "\n"; 
    } 

    public Long getNodeId() { 
     return nodeId; 
    } 

    public void setNodeId(Long nodeId) { 
     this.nodeId = nodeId; 
    } 

    public String getKey() { 
     return key; 
    } 

    public void setKey(String key) { 
     this.key = key; 
    } 

    public String getMdate() { 
     return mdate; 
    } 

    public void setMdate(String mdate) { 
     this.mdate = mdate; 
    } 

    public String getPubltype() { 
     return publtype; 
    } 

    public void setPubltype(String publtype) { 
     this.publtype = publtype; 
    } 

    public String getReviewid() { 
     return reviewid; 
    } 

    public void setReviewid(String reviewid) { 
     this.reviewid = reviewid; 
    } 

    public String getRating() { 
     return rating; 
    } 

    public void setRating(String rating) { 
     this.rating = rating; 
    } 

    public String getType() { 
     return type; 
    } 

    public void setType(String type) { 
     this.type = type; 
    } 

    public String getEditor() { 
     return editor; 
    } 

    public void setEditor(String editor) { 
     this.editor = editor; 
    } 

    public String getTitle() { 
     return title; 
    } 

    public void setTitle(String title) { 
     this.title = title; 
    } 

    public String getBooktitle() { 
     return booktitle; 
    } 

    public void setBooktitle(String booktitle) { 
     this.booktitle = booktitle; 
    } 

    public String getPages() { 
     return pages; 
    } 

    public void setPages(String pages) { 
     this.pages = pages; 
    } 

    public String getYear() { 
     return year; 
    } 

    public void setYear(String year) { 
     this.year = year; 
    } 

    public String getAddress() { 
     return address; 
    } 

    public void setAddress(String address) { 
     this.address = address; 
    } 

    public String getJournal() { 
     return journal; 
    } 

    public void setJournal(String journal) { 
     this.journal = journal; 
    } 

    public String getVolume() { 
     return volume; 
    } 

    public void setVolume(String volume) { 
     this.volume = volume; 
    } 

    public String getNumber() { 
     return number; 
    } 

    public void setNumber(String number) { 
     this.number = number; 
    } 

    public String getMonth() { 
     return month; 
    } 

    public void setMonth(String month) { 
     this.month = month; 
    } 

    public String getUrl() { 
     return url; 
    } 

    public void setUrl(String url) { 
     this.url = url; 
    } 

    public String getEe() { 
     return ee; 
    } 

    public void setEe(String ee) { 
     this.ee = ee; 
    } 

    public String getCdrom() { 
     return cdrom; 
    } 

    public void setCdrom(String cdrom) { 
     this.cdrom = cdrom; 
    } 

    public String getCite() { 
     return cite; 
    } 

    public void setCite(String cite) { 
     this.cite = cite; 
    } 

    public String getPublisher() { 
     return publisher; 
    } 

    public void setPublisher(String publisher) { 
     this.publisher = publisher; 
    } 

    public String getNote() { 
     return note; 
    } 

    public void setNote(String note) { 
     this.note = note; 
    } 

    public String getCrossref() { 
     return crossref; 
    } 

    public void setCrossref(String crossref) { 
     this.crossref = crossref; 
    } 

    public String getIsbn() { 
     return isbn; 
    } 

    public void setIsbn(String isbn) { 
     this.isbn = isbn; 
    } 

    public String getSeries() { 
     return series; 
    } 

    public void setSeries(String series) { 
     this.series = series; 
    } 

    public String getSchool() { 
     return school; 
    } 

    public void setSchool(String school) { 
     this.school = school; 
    } 

    public String getChapter() { 
     return chapter; 
    } 

    public void setChapter(String chapter) { 
     this.chapter = chapter; 
    } 

}

該應用程序使用Maven作爲構建自動化工具，這裏的pom.xml：

<?xml version="1.0" encoding="UTF-8"?> 
<project xmlns="http://maven.apache.org/POM/4.0.0" 
      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
      xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 

    <modelVersion>4.0.0</modelVersion> 

    <groupId>com.dblp</groupId> 
    <artifactId>graphdbcreator</artifactId> 
    <version>0.1.0</version> 

    <parent> 
     <groupId>org.springframework.boot</groupId> 
     <artifactId>spring-boot-starter-parent</artifactId> 
     <version>1.2.2.RELEASE</version> 
    </parent> 

    <dependencies> 
     <dependency> 
      <groupId>org.springframework.boot</groupId> 
      <artifactId>spring-boot-starter</artifactId> 
     </dependency> 
     <dependency> 
      <groupId>org.springframework</groupId> 
      <artifactId>spring-context</artifactId> 
     </dependency> 
     <dependency> 
      <groupId>org.springframework</groupId> 
      <artifactId>spring-tx</artifactId> 
     </dependency> 
     <dependency> 
      <groupId>org.springframework.data</groupId> 
      <artifactId>spring-data-neo4j</artifactId> 
     </dependency> 
     <dependency> 
      <groupId>org.hibernate</groupId> 
      <artifactId>hibernate-validator</artifactId> 
     </dependency> 
     <dependency> 
      <groupId>javax.el</groupId> 
      <artifactId>javax.el-api</artifactId> 
      <version>2.2.4</version> 
     </dependency> 
     <dependency> 
      <groupId>xerces</groupId> 
      <artifactId>xercesImpl</artifactId> 
      <version>2.8.0</version> 
     </dependency> 
    </dependencies> 

    <build> 
     <plugins> 
      <plugin> 
       <groupId>org.springframework.boot</groupId> 
       <artifactId>spring-boot-maven-plugin</artifactId> 
      </plugin> 
     </plugins> 
    </build> 

    <repositories> 
     <repository> 
      <id>spring-releases</id> 
      <name>Spring Releases</name> 
      <url>https://repo.spring.io/libs-release</url> 
     </repository> 
     <repository> 
      <id>neo4j</id> 
      <name>Neo4j</name> 
      <url>http://m2.neo4j.org/</url> 
     </repository> 
    </repositories> 

</project>

這裏是中央部分錯誤消息：

java.lang.OutOfMemoryError: GC overhead limit exceeded 
    at org.neo4j.collection.primitive.PrimitiveIntCollections.toPrimitiveIterator(PrimitiveIntCollections.java:678) 
    at org.neo4j.kernel.impl.api.StateHandlingStatementOperations.nodeGetLabels(StateHandlingStatementOperations.java:188) 
    at org.neo4j.kernel.impl.api.ConstraintEnforcingEntityOperations.nodeSetProperty(ConstraintEnforcingEntityOperations.java:85) 
    at org.neo4j.kernel.impl.api.LockingStatementOperations.nodeSetProperty(LockingStatementOperations.java:280) 
    at org.neo4j.kernel.impl.api.OperationsFacade.nodeSetProperty(OperationsFacade.java:551) 
    at org.neo4j.kernel.impl.core.NodeProxy.setProperty(NodeProxy.java:254) 
    at org.springframework.data.neo4j.fieldaccess.PropertyFieldAccessorFactory$PropertyFieldAccessor.setValue(PropertyFieldAccessorFactory.java:76) 
    at org.springframework.data.neo4j.fieldaccess.DefaultEntityState.setValue(DefaultEntityState.java:113) 
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.setEntityStateValue(SourceStateTransmitter.java:70) 
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.access$100(SourceStateTransmitter.java:40) 
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter$3.doWithPersistentProperty(SourceStateTransmitter.java:105) 
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter$3.doWithPersistentProperty(SourceStateTransmitter.java:102) 
    at org.springframework.data.mapping.model.BasicPersistentEntity.doWithProperties(BasicPersistentEntity.java:307) 
    at org.springframework.data.neo4j.support.mapping.SourceStateTransmitter.copyPropertiesTo(SourceStateTransmitter.java:102) 
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityConverterImpl.write(Neo4jEntityConverterImpl.java:167) 
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityPersister$CachedConverter.write(Neo4jEntityPersister.java:179) 
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityPersister.persist(Neo4jEntityPersister.java:243) 
    at org.springframework.data.neo4j.support.mapping.Neo4jEntityPersister.persist(Neo4jEntityPersister.java:231) 
    at org.springframework.data.neo4j.support.Neo4jTemplate.save(Neo4jTemplate.java:357) 
    at org.springframework.data.neo4j.support.Neo4jTemplate.save(Neo4jTemplate.java:351) 
    at org.springframework.data.neo4j.repository.AbstractGraphRepository.save(AbstractGraphRepository.java:91) 
    at sun.reflect.GeneratedMethodAccessor28.invoke(Unknown Source) 
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 
    at java.lang.reflect.Method.invoke(Method.java:497) 
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.executeMethodOn(RepositoryFactorySupport.java:416) 
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.doInvoke(RepositoryFactorySupport.java:401) 
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.invoke(RepositoryFactorySupport.java:373) 
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) 
    at org.springframework.data.repository.core.support.RepositoryFactorySupport$DefaultMethodInvokingMethodInterceptor.invoke(RepositoryFactorySupport.java:486) 
    at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:179) 
    at org.springframework.transaction.interceptor.TransactionInterceptor$1.proceedWithInvocation(TransactionInterceptor.java:99) 
    at org.springframework.transaction.interceptor.TransactionAspectSupport.invokeWithinTransaction(TransactionAspectSupport.java:281)

來源

2015-03-31 GCOverhead

我們有同樣的問題用這個，有什麼不對的新添加新節點和關係的時候。嘗試導入使用.csv文件< - 這使我們哭了創造，但最終解決了這個問題 – nafas 2015-03-31 12:32:04

您的交易也是大。

嘗試創建一個僅導入一批一千個出版物，並用@Transactional註解它（我們在該方法中使用手動交易）。然後從主類調用該方法（獲取爲spring-tx處理注入的服務），並使用批量的1000個發佈。

或者，在你的循環代碼

tx = db.beginTx(); 
while() { 

// do stuff 

if (++counter % 1000 == 0) { 
    tx.success(); tx.close(); 
    tx = db.beginTx(); 
} 
} 
    tx.success(); tx.close();

來源

2015-03-31 13:23:27

彈簧數據Neo4j超出了GC開銷限制

回答

相關問題