Compare commits
No commits in common. '1e8b779a0fc59d83fe7ea95d01b07b8797d1dce3' and 'a64fba0cb1d7d68493c804dc0620a4e06f6c95db' have entirely different histories.
1e8b779a0f
...
a64fba0cb1
@ -1,11 +0,0 @@ |
||||
package me.bearns.fias.service; |
||||
|
||||
import me.bearns.fias.util.UnmarshallerParameters; |
||||
|
||||
import java.io.InputStream; |
||||
import java.util.function.Predicate; |
||||
|
||||
public interface StreamSaver { |
||||
|
||||
public void process(InputStream is, UnmarshallerParameters conf, Predicate filter); |
||||
} |
||||
@ -1,95 +0,0 @@ |
||||
package me.bearns.fias.service; |
||||
|
||||
import me.bearns.fias.util.UnmarshallerParameters; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
import javax.xml.bind.JAXBContext; |
||||
import javax.xml.bind.JAXBException; |
||||
import javax.xml.bind.Unmarshaller; |
||||
import javax.xml.stream.XMLEventReader; |
||||
import javax.xml.stream.XMLInputFactory; |
||||
import javax.xml.stream.XMLStreamException; |
||||
import javax.xml.stream.events.StartElement; |
||||
import javax.xml.stream.events.XMLEvent; |
||||
import java.io.InputStream; |
||||
import java.util.function.Predicate; |
||||
|
||||
@Component |
||||
public class StreamSaverImpl implements StreamSaver{ |
||||
@Override |
||||
public void process(InputStream is, UnmarshallerParameters conf, Predicate filter) { |
||||
|
||||
try { |
||||
// create xml event reader for input stream
|
||||
final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); |
||||
final XMLEventReader xmlEventReader = xmlInputFactory.createXMLEventReader(is); |
||||
|
||||
|
||||
// initialize jaxb
|
||||
final JAXBContext context = JAXBContext.newInstance(conf.getParentCls()); |
||||
final Unmarshaller unmarshaller = context.createUnmarshaller(); |
||||
|
||||
XMLEvent e = null; |
||||
|
||||
long countRead=0, countWrite=0, countErrors=0; |
||||
|
||||
// loop though the xml stream
|
||||
while (true) { |
||||
|
||||
if (!((e = xmlEventReader.peek()) != null)) break; |
||||
|
||||
countRead++; |
||||
|
||||
// check the event is a Document start element
|
||||
if (e.isStartElement() && ((StartElement) e).getName().equals(conf.getQName())) { |
||||
|
||||
Object obj; |
||||
try { |
||||
// unmarshall the document
|
||||
obj = unmarshaller.unmarshal(xmlEventReader, conf.getCls()).getValue(); |
||||
|
||||
} catch (Exception ex) { |
||||
countErrors++; |
||||
//log.error("Unmarshalling error {} in {}", ex.getMessage(), file.getName());
|
||||
|
||||
/*final Iterator attributes = ((StartElement) e).getAttributes(); |
||||
attributes.forEachRemaining( |
||||
(a) -> log.error("Object dump: {} {}", ((Attribute) a).getName(), ((Attribute) a).getValue()) |
||||
);*/ |
||||
|
||||
continue; |
||||
} |
||||
|
||||
//if(countRead%100000==0) log.info("Read {} item from {}", countRead, file.getName());
|
||||
|
||||
//Region filter
|
||||
if(filter != null && !filter.test(obj)) continue; |
||||
|
||||
//Save to repository
|
||||
conf.getRepository().save(obj); |
||||
|
||||
/*if(++countWrite%10000==0) { |
||||
repository.flush(); |
||||
log.info("Write {} items to repository.", countWrite); |
||||
}*/ |
||||
|
||||
} else { |
||||
xmlEventReader.next(); |
||||
} |
||||
} |
||||
} catch (XMLStreamException ex) { |
||||
ex.printStackTrace(); //createXMLEventReader & peak
|
||||
} catch (JAXBException ex) { |
||||
ex.printStackTrace(); //createUnmarshaller();
|
||||
} |
||||
|
||||
//write to DB from cache
|
||||
conf.getRepository().flush(); |
||||
|
||||
/*log.info("SUCSESS: read={}, write={}", countRead, countWrite); |
||||
if(countErrors > 0) { |
||||
log.warn("Unmarshalling errors suppressed: {}", countErrors); |
||||
log.warn("Check data consistency in {}", file.getName()); |
||||
}*/ |
||||
} |
||||
} |
||||
@ -1,82 +0,0 @@ |
||||
package me.bearns.fias.util; |
||||
|
||||
import me.bearns.fias.repository.FiasVersionRepository; |
||||
import org.springframework.data.jpa.repository.JpaRepository; |
||||
import org.springframework.stereotype.Component; |
||||
|
||||
import javax.xml.namespace.QName; |
||||
import java.util.ArrayList; |
||||
import java.util.List; |
||||
|
||||
@Component |
||||
public class Catalog { |
||||
|
||||
private final List<Item> prefixList = new ArrayList<>(); |
||||
|
||||
public Catalog(FiasVersionRepository FIASJPA_REPOSITORY) { |
||||
prefixList.add( |
||||
new Item( |
||||
"AS_ADDROBJ_", |
||||
null, |
||||
new QName("","Object"), |
||||
null, |
||||
null |
||||
) |
||||
); |
||||
prefixList.add( |
||||
new Item( |
||||
"AS_HOUSE_", |
||||
null, |
||||
new QName("","House"), |
||||
null, |
||||
null |
||||
) |
||||
); |
||||
} |
||||
|
||||
public Item getByPrefix(String s){ |
||||
if(s!=null) { |
||||
|
||||
for (Item item : prefixList) { |
||||
if (s.startsWith(item.prefix)) return item; |
||||
} |
||||
} |
||||
return null; |
||||
} |
||||
|
||||
public class Item implements UnmarshallerParameters { |
||||
private Item(String prefix, JpaRepository repository, QName qName, Class cls, Class parentCls) { |
||||
this.prefix = prefix; |
||||
this.repository = repository; |
||||
this.qName = qName; |
||||
this.cls = cls; |
||||
this.parentCls = parentCls; |
||||
} |
||||
private final String prefix; |
||||
private final JpaRepository repository; |
||||
private final QName qName; |
||||
private final Class cls; |
||||
private final Class parentCls; |
||||
|
||||
|
||||
@Override |
||||
public JpaRepository getRepository() { |
||||
return repository; |
||||
} |
||||
|
||||
@Override |
||||
public QName getQName() { |
||||
return qName; |
||||
} |
||||
|
||||
@Override |
||||
public Class getCls() { |
||||
return cls; |
||||
} |
||||
|
||||
@Override |
||||
public Class getParentCls() { |
||||
return parentCls; |
||||
} |
||||
} |
||||
} |
||||
@ -1,12 +0,0 @@ |
||||
package me.bearns.fias.util; |
||||
|
||||
import org.springframework.data.jpa.repository.JpaRepository; |
||||
|
||||
import javax.xml.namespace.QName; |
||||
|
||||
public interface UnmarshallerParameters { |
||||
public JpaRepository getRepository(); |
||||
public QName getQName(); |
||||
public Class getCls(); |
||||
public Class getParentCls(); |
||||
} |
||||
Loading…
Reference in new issue