|
|
|
@ -1,5 +1,6 @@ |
|
|
|
|
package me.bearns.fias.service; |
|
|
|
|
|
|
|
|
|
import lombok.extern.slf4j.Slf4j; |
|
|
|
|
import me.bearns.fias.exceptions.CommonException; |
|
|
|
|
import me.bearns.fias.exceptions.UnmarshallingException; |
|
|
|
|
import me.bearns.fias.helpers.UnmarshallerParameters; |
|
|
|
@ -11,16 +12,21 @@ import javax.xml.bind.Unmarshaller; |
|
|
|
|
import javax.xml.stream.XMLEventReader; |
|
|
|
|
import javax.xml.stream.XMLInputFactory; |
|
|
|
|
import javax.xml.stream.XMLStreamException; |
|
|
|
|
import javax.xml.stream.events.Attribute; |
|
|
|
|
import javax.xml.stream.events.StartElement; |
|
|
|
|
import javax.xml.stream.events.XMLEvent; |
|
|
|
|
import java.io.InputStream; |
|
|
|
|
import java.util.Iterator; |
|
|
|
|
import java.util.function.Predicate; |
|
|
|
|
|
|
|
|
|
@Slf4j |
|
|
|
|
@Component |
|
|
|
|
public class StreamSaverImpl implements StreamSaver{ |
|
|
|
|
@Override |
|
|
|
|
public void process(InputStream is, UnmarshallerParameters conf, Predicate filter) throws CommonException { |
|
|
|
|
|
|
|
|
|
log.debug("Init unmarshaller"); |
|
|
|
|
|
|
|
|
|
try { |
|
|
|
|
// create xml event reader for input stream
|
|
|
|
|
final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); |
|
|
|
@ -35,6 +41,8 @@ public class StreamSaverImpl implements StreamSaver{ |
|
|
|
|
|
|
|
|
|
long countRead=0, countWrite=0, countErrors=0; |
|
|
|
|
|
|
|
|
|
log.debug("Start reading stream"); |
|
|
|
|
|
|
|
|
|
// loop though the xml stream
|
|
|
|
|
while ((e = xmlEventReader.peek()) != null) { |
|
|
|
|
|
|
|
|
@ -50,12 +58,16 @@ public class StreamSaverImpl implements StreamSaver{ |
|
|
|
|
|
|
|
|
|
} catch (Exception ex) { |
|
|
|
|
countErrors++; |
|
|
|
|
//log.error("Unmarshalling error {} in {}", ex.getMessage(), file.getName());
|
|
|
|
|
|
|
|
|
|
/*final Iterator attributes = ((StartElement) e).getAttributes(); |
|
|
|
|
attributes.forEachRemaining( |
|
|
|
|
(a) -> log.error("Object dump: {} {}", ((Attribute) a).getName(), ((Attribute) a).getValue()) |
|
|
|
|
);*/ |
|
|
|
|
log.error("Unmarshalling error[{}] {}", countErrors, ex.getMessage()); |
|
|
|
|
|
|
|
|
|
if(log.isTraceEnabled()) { |
|
|
|
|
//debug print
|
|
|
|
|
final Iterator attributes = ((StartElement) e).getAttributes(); |
|
|
|
|
attributes.forEachRemaining( |
|
|
|
|
(a) -> log.error("Object dump: {} {}", ((Attribute) a).getName(), ((Attribute) a).getValue()) |
|
|
|
|
); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
continue; |
|
|
|
|
} |
|
|
|
@ -67,8 +79,9 @@ public class StreamSaverImpl implements StreamSaver{ |
|
|
|
|
|
|
|
|
|
//Save to repository
|
|
|
|
|
conf.getRepository().save(obj); |
|
|
|
|
countWrite++; |
|
|
|
|
|
|
|
|
|
/*if(++countWrite%10000==0) { |
|
|
|
|
/*if(countWrite%10000==0) { |
|
|
|
|
repository.flush(); |
|
|
|
|
log.info("Write {} items to repository.", countWrite); |
|
|
|
|
}*/ |
|
|
|
@ -77,21 +90,21 @@ public class StreamSaverImpl implements StreamSaver{ |
|
|
|
|
xmlEventReader.next(); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
log.info("SUCSESS: read={}, write={}", countRead, countWrite); |
|
|
|
|
if(countErrors > 0) log.warn("Completed with errors, problems count: {}", countErrors); |
|
|
|
|
|
|
|
|
|
} catch (XMLStreamException ex) { |
|
|
|
|
//todo log
|
|
|
|
|
throw new UnmarshallingException(ex); //createXMLEventReader & peak
|
|
|
|
|
log.error("Caught XMLStreamException (createXMLEventReader or peak methods)"); |
|
|
|
|
throw new UnmarshallingException(ex); |
|
|
|
|
} catch (JAXBException ex) { |
|
|
|
|
//todo log
|
|
|
|
|
throw new UnmarshallingException(ex); //createUnmarshaller();
|
|
|
|
|
log.error("Caught JAXBException on createUnmarshaller()"); |
|
|
|
|
throw new UnmarshallingException(ex); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
//write to DB from cache
|
|
|
|
|
conf.getRepository().flush(); |
|
|
|
|
|
|
|
|
|
/*log.info("SUCSESS: read={}, write={}", countRead, countWrite); |
|
|
|
|
if(countErrors > 0) { |
|
|
|
|
log.warn("Unmarshalling errors suppressed: {}", countErrors); |
|
|
|
|
log.warn("Check data consistency in {}", file.getName()); |
|
|
|
|
}*/ |
|
|
|
|
log.debug("Repository flush completed"); |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|