Compare commits

..

No commits in common. '694bafcd986b138cd1ce60c54e69e0d500cfb897' and '6197956d4fd7f147e496b7572d4f3c78601311d9' have entirely different histories.

  1. 2
      src/main/java/me/bearns/fias/config/ApplicationConfig.java
  2. 1
      src/main/java/me/bearns/fias/service/DownloaderImpl.java
  3. 39
      src/main/java/me/bearns/fias/service/StreamSaverImpl.java
  4. 6
      src/main/resources/application-default.properties
  5. 1
      src/main/resources/application.properties
  6. 12
      src/test/java/me/bearns/fias/TransactionsTest.java
  7. 2
      src/test/java/me/bearns/fias/VersionRepositoryTests.java
  8. 6
      src/test/java/me/bearns/fias/helper/TransactionalSaveHelper.java

@ -20,7 +20,7 @@ import javax.sql.DataSource;
@Configuration
@EnableJpaRepositories("me.bearns.fias.repository")
@EnableTransactionManagement
//@PropertySource(value = "application-{profile}.properties", ignoreResourceNotFound = true)
@PropertySource(value = "application.properties", ignoreResourceNotFound = true)
class ApplicationConfig {
@Autowired

@ -35,6 +35,7 @@ public class DownloaderImpl implements Downloader {
if(feature != start) return feature;
pool.submit(() -> {
//todo
final URL conn;
final Path path;

@ -1,6 +1,5 @@
package me.bearns.fias.service;
import lombok.extern.slf4j.Slf4j;
import me.bearns.fias.exceptions.CommonException;
import me.bearns.fias.exceptions.UnmarshallingException;
import me.bearns.fias.helpers.UnmarshallerParameters;
@ -12,21 +11,16 @@ import javax.xml.bind.Unmarshaller;
import javax.xml.stream.XMLEventReader;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.events.Attribute;
import javax.xml.stream.events.StartElement;
import javax.xml.stream.events.XMLEvent;
import java.io.InputStream;
import java.util.Iterator;
import java.util.function.Predicate;
@Slf4j
@Component
public class StreamSaverImpl implements StreamSaver{
@Override
public void process(InputStream is, UnmarshallerParameters conf, Predicate filter) throws CommonException {
log.debug("Init unmarshaller");
try {
// create xml event reader for input stream
final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance();
@ -41,8 +35,6 @@ public class StreamSaverImpl implements StreamSaver{
long countRead=0, countWrite=0, countErrors=0;
log.debug("Start reading stream");
// loop though the xml stream
while ((e = xmlEventReader.peek()) != null) {
@ -58,16 +50,12 @@ public class StreamSaverImpl implements StreamSaver{
} catch (Exception ex) {
countErrors++;
//log.error("Unmarshalling error {} in {}", ex.getMessage(), file.getName());
log.error("Unmarshalling error[{}] {}", countErrors, ex.getMessage());
if(log.isTraceEnabled()) {
//debug print
final Iterator attributes = ((StartElement) e).getAttributes();
/*final Iterator attributes = ((StartElement) e).getAttributes();
attributes.forEachRemaining(
(a) -> log.error("Object dump: {} {}", ((Attribute) a).getName(), ((Attribute) a).getValue())
);
}
);*/
continue;
}
@ -79,9 +67,8 @@ public class StreamSaverImpl implements StreamSaver{
//Save to repository
conf.getRepository().save(obj);
countWrite++;
/*if(countWrite%10000==0) {
/*if(++countWrite%10000==0) {
repository.flush();
log.info("Write {} items to repository.", countWrite);
}*/
@ -90,21 +77,21 @@ public class StreamSaverImpl implements StreamSaver{
xmlEventReader.next();
}
}
log.info("SUCSESS: read={}, write={}", countRead, countWrite);
if(countErrors > 0) log.warn("Completed with errors, problems count: {}", countErrors);
} catch (XMLStreamException ex) {
log.error("Caught XMLStreamException (createXMLEventReader or peak methods)");
throw new UnmarshallingException(ex);
//todo log
throw new UnmarshallingException(ex); //createXMLEventReader & peak
} catch (JAXBException ex) {
log.error("Caught JAXBException on createUnmarshaller()");
throw new UnmarshallingException(ex);
//todo log
throw new UnmarshallingException(ex); //createUnmarshaller();
}
//write to DB from cache
conf.getRepository().flush();
log.debug("Repository flush completed");
/*log.info("SUCSESS: read={}, write={}", countRead, countWrite);
if(countErrors > 0) {
log.warn("Unmarshalling errors suppressed: {}", countErrors);
log.warn("Check data consistency in {}", file.getName());
}*/
}
}

@ -1,6 +0,0 @@
spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/databasename
spring.datasource.driverClassName=org.postgresql.Driver
spring.datasource.username=sa
spring.datasource.password=password
spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
logging.level.root=INFO

@ -3,4 +3,3 @@ spring.datasource.driverClassName=org.h2.Driver
spring.datasource.username=sa
spring.datasource.password=password
spring.jpa.database-platform=org.hibernate.dialect.H2Dialect
logging.level.root=DEBUG

@ -1,6 +1,5 @@
package me.bearns.fias;
import lombok.extern.slf4j.Slf4j;
import me.bearns.fias.domain.FiasVersion;
import me.bearns.fias.exceptions.UnmarshallingException;
import me.bearns.fias.helper.TransactionalSaveHelper;
@ -12,7 +11,6 @@ import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.Import;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.transaction.annotation.Transactional;
@ -21,8 +19,6 @@ import java.util.List;
import static me.bearns.fias.helper.TransactionalSaveHelper.FIRST;
import static me.bearns.fias.helper.TransactionalSaveHelper.NEXT;
@Slf4j
@ActiveProfiles("test")
@RunWith(SpringRunner.class)
@SpringBootTest
@Import(TransactionalSaveHelper.class)
@ -36,14 +32,12 @@ public class TransactionsTest {
@Before
public void before(){
log.info("Configure test components");
helper.setRepository(repository);
}
@Test
public void transactionTest(){
log.info("Run transaction test");
helper.addFirst();
try {
@ -51,26 +45,22 @@ public class TransactionsTest {
assert false;
} catch (Exception e) {
//just as plained
log.info("Caught exception, continue");
assert true;
}
final List<FiasVersion> versions = repository.findAll();
versions.stream().map(FiasVersion::toString).forEach(log::info);
versions.stream().map(FiasVersion::getVersionId).forEach(System.out::println);
assert versions.stream().mapToLong(FiasVersion::getVersionId).anyMatch(v -> FIRST == v);
log.info("find {}", FIRST);
assert versions.stream().mapToLong(FiasVersion::getVersionId).noneMatch(v -> NEXT == v);
log.info("not find {}", NEXT);
}
@After
public void clear(){
repository.deleteAll();
log.info("Clear FiasVersion repo");
}

@ -26,9 +26,7 @@ import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
import org.springframework.boot.test.autoconfigure.orm.jpa.TestEntityManager;
import org.springframework.test.context.ActiveProfiles;
@ActiveProfiles("test")
@DataJpaTest
public class VersionRepositoryTests {
@Autowired

@ -1,6 +1,5 @@
package me.bearns.fias.helper;
import lombok.extern.slf4j.Slf4j;
import me.bearns.fias.domain.FiasVersion;
import me.bearns.fias.exceptions.CommonException;
import me.bearns.fias.exceptions.UnmarshallingException;
@ -8,7 +7,6 @@ import me.bearns.fias.repository.FiasVersionRepository;
import org.springframework.boot.test.context.TestComponent;
import org.springframework.transaction.annotation.Transactional;
@Slf4j
@TestComponent
public class TransactionalSaveHelper {
@ -24,12 +22,12 @@ public class TransactionalSaveHelper {
@Transactional(rollbackFor = Exception.class)
public void addNext() throws CommonException {
final FiasVersion nextVersion = new FiasVersion(NEXT, null, null, null);
log.info("Saved {}",repository.save(nextVersion));
repository.save(nextVersion);
throw new UnmarshallingException("test");
}
public void addFirst() {
final FiasVersion version = new FiasVersion(FIRST, null, null, null);
log.info("Saved {}", repository.save(version));
repository.save(version);
}
}

Loading…
Cancel
Save