diff --git a/pom.xml b/pom.xml index fb83473..a4edc34 100644 --- a/pom.xml +++ b/pom.xml @@ -19,6 +19,12 @@ + + org.projectlombok + lombok + 1.18.12 + provided + org.springframework.boot spring-boot-starter-data-jpa diff --git a/src/main/java/me/bearns/fias/domain/Addrobj.java b/src/main/java/me/bearns/fias/domain/Addrobj.java index 49640a5..ae21499 100644 --- a/src/main/java/me/bearns/fias/domain/Addrobj.java +++ b/src/main/java/me/bearns/fias/domain/Addrobj.java @@ -1,6 +1,9 @@ package me.bearns.fias.domain; +import lombok.EqualsAndHashCode; +import lombok.ToString; + import javax.persistence.*; import javax.xml.bind.annotation.*; import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; @@ -14,6 +17,8 @@ import java.util.GregorianCalendar; import java.util.Objects; import java.util.UUID; +@EqualsAndHashCode +@ToString @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "") @Entity @@ -203,68 +208,4 @@ public class Addrobj implements Serializable { this.aoid = aoid; return this; } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Addrobj addrobj = (Addrobj) o; - - if (livestatus != addrobj.livestatus) return false; - if (divtype != addrobj.divtype) return false; - if (!aoid.equals(addrobj.aoid)) return false; - if (!aoguid.equals(addrobj.aoguid)) return false; - if (formalname != null ? !formalname.equals(addrobj.formalname) : addrobj.formalname != null) return false; - if (regioncode != null ? !regioncode.equals(addrobj.regioncode) : addrobj.regioncode != null) return false; - if (autocode != null ? !autocode.equals(addrobj.autocode) : addrobj.autocode != null) return false; - if (areacode != null ? !areacode.equals(addrobj.areacode) : addrobj.areacode != null) return false; - if (citycode != null ? !citycode.equals(addrobj.citycode) : addrobj.citycode != null) return false; - if (ctarcode != null ? !ctarcode.equals(addrobj.ctarcode) : addrobj.ctarcode != null) return false; - if (placecode != null ? !placecode.equals(addrobj.placecode) : addrobj.placecode != null) return false; - if (plancode != null ? !plancode.equals(addrobj.plancode) : addrobj.plancode != null) return false; - if (streetcode != null ? !streetcode.equals(addrobj.streetcode) : addrobj.streetcode != null) return false; - if (extrcode != null ? !extrcode.equals(addrobj.extrcode) : addrobj.extrcode != null) return false; - if (sextcode != null ? !sextcode.equals(addrobj.sextcode) : addrobj.sextcode != null) return false; - if (offname != null ? !offname.equals(addrobj.offname) : addrobj.offname != null) return false; - if (postalcode != null ? !postalcode.equals(addrobj.postalcode) : addrobj.postalcode != null) return false; - if (ifnsfl != null ? !ifnsfl.equals(addrobj.ifnsfl) : addrobj.ifnsfl != null) return false; - if (terrifnsfl != null ? !terrifnsfl.equals(addrobj.terrifnsfl) : addrobj.terrifnsfl != null) return false; - if (ifnsul != null ? !ifnsul.equals(addrobj.ifnsul) : addrobj.ifnsul != null) return false; - if (terrifnsul != null ? !terrifnsul.equals(addrobj.terrifnsul) : addrobj.terrifnsul != null) return false; - if (okato != null ? !okato.equals(addrobj.okato) : addrobj.okato != null) return false; - if (oktmo != null ? !oktmo.equals(addrobj.oktmo) : addrobj.oktmo != null) return false; - if (updatedate != null ? !updatedate.equals(addrobj.updatedate) : addrobj.updatedate != null) return false; - if (shortname != null ? !shortname.equals(addrobj.shortname) : addrobj.shortname != null) return false; - if (aolevel != null ? !aolevel.equals(addrobj.aolevel) : addrobj.aolevel != null) return false; - if (parentguid != null ? !parentguid.equals(addrobj.parentguid) : addrobj.parentguid != null) return false; - if (previd != null ? !previd.equals(addrobj.previd) : addrobj.previd != null) return false; - if (nextid != null ? !nextid.equals(addrobj.nextid) : addrobj.nextid != null) return false; - if (code != null ? !code.equals(addrobj.code) : addrobj.code != null) return false; - if (plaincode != null ? !plaincode.equals(addrobj.plaincode) : addrobj.plaincode != null) return false; - if (actstatus != null ? !actstatus.equals(addrobj.actstatus) : addrobj.actstatus != null) return false; - if (centstatus != null ? !centstatus.equals(addrobj.centstatus) : addrobj.centstatus != null) return false; - if (operstatus != null ? !operstatus.equals(addrobj.operstatus) : addrobj.operstatus != null) return false; - if (currstatus != null ? !currstatus.equals(addrobj.currstatus) : addrobj.currstatus != null) return false; - if (startdate != null ? !startdate.equals(addrobj.startdate) : addrobj.startdate != null) return false; - if (enddate != null ? !enddate.equals(addrobj.enddate) : addrobj.enddate != null) return false; - return normdoc != null ? normdoc.equals(addrobj.normdoc) : addrobj.normdoc == null; - - } - - @Override - public int hashCode() { - return Objects.hashCode(aoid); - } - - @Override - public String toString() { - return "Addrobj{" + - "aoguid='" + aoguid + '\'' + - ", areacode='" + areacode + '\'' + - ", citycode='" + citycode + '\'' + - ", shortname='" + shortname + '\'' + - ", actstatus=" + actstatus + - '}'; - } } \ No newline at end of file diff --git a/src/main/java/me/bearns/fias/domain/FiasVersion.java b/src/main/java/me/bearns/fias/domain/FiasVersion.java index 4d3bb09..0082ed0 100644 --- a/src/main/java/me/bearns/fias/domain/FiasVersion.java +++ b/src/main/java/me/bearns/fias/domain/FiasVersion.java @@ -1,10 +1,13 @@ package me.bearns.fias.domain; +import lombok.ToString; + import javax.persistence.Column; import javax.persistence.Id; import javax.persistence.Entity; import javax.persistence.Table; +@ToString @Entity @Table(name = "version") public class FiasVersion { diff --git a/src/main/java/me/bearns/fias/domain/House.java b/src/main/java/me/bearns/fias/domain/House.java index c96bfc2..00fea95 100644 --- a/src/main/java/me/bearns/fias/domain/House.java +++ b/src/main/java/me/bearns/fias/domain/House.java @@ -1,5 +1,8 @@ package me.bearns.fias.domain; +import lombok.EqualsAndHashCode; +import lombok.ToString; + import javax.persistence.*; import javax.xml.bind.annotation.*; import javax.xml.datatype.DatatypeConfigurationException; @@ -9,9 +12,10 @@ import java.io.Serializable; import java.math.BigInteger; import java.util.Calendar; import java.util.GregorianCalendar; -import java.util.Objects; import java.util.UUID; +@EqualsAndHashCode +@ToString @XmlAccessorType(XmlAccessType.FIELD) @XmlType(name = "") @Entity @@ -92,8 +96,6 @@ public class House implements Serializable { @XmlAttribute(name = "DIVTYPE", required = true) protected int divtype; - // jhipster-needle-entity-add-field - JHipster will add fields here, do not remove - public House aoguid(String aoguid) { this.aoguid = UUID.fromString(aoguid); return this; @@ -150,54 +152,4 @@ public class House implements Serializable { public void setHouseguid(UUID houseguid) { this.houseguid = houseguid; } - // jhipster-needle-entity-add-getters-setters - JHipster will add getters and setters here, do not remove - - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - House house = (House) o; - - if (divtype != house.divtype) return false; - if (!houseid.equals(house.houseid)) return false; - if (aoguid != null ? !aoguid.equals(house.aoguid) : house.aoguid != null) return false; - if (houseguid != null ? !houseguid.equals(house.houseguid) : house.houseguid != null) return false; - if (postalcode != null ? !postalcode.equals(house.postalcode) : house.postalcode != null) return false; - if (regioncode != null ? !regioncode.equals(house.regioncode) : house.regioncode != null) return false; - if (ifnsfl != null ? !ifnsfl.equals(house.ifnsfl) : house.ifnsfl != null) return false; - if (terrifnsfl != null ? !terrifnsfl.equals(house.terrifnsfl) : house.terrifnsfl != null) return false; - if (ifnsul != null ? !ifnsul.equals(house.ifnsul) : house.ifnsul != null) return false; - if (terrifnsul != null ? !terrifnsul.equals(house.terrifnsul) : house.terrifnsul != null) return false; - if (okato != null ? !okato.equals(house.okato) : house.okato != null) return false; - if (oktmo != null ? !oktmo.equals(house.oktmo) : house.oktmo != null) return false; - if (updatedate != null ? !updatedate.equals(house.updatedate) : house.updatedate != null) return false; - if (housenum != null ? !housenum.equals(house.housenum) : house.housenum != null) return false; - if (eststatus != null ? !eststatus.equals(house.eststatus) : house.eststatus != null) return false; - if (buildnum != null ? !buildnum.equals(house.buildnum) : house.buildnum != null) return false; - if (strucnum != null ? !strucnum.equals(house.strucnum) : house.strucnum != null) return false; - if (strstatus != null ? !strstatus.equals(house.strstatus) : house.strstatus != null) return false; - if (startdate != null ? !startdate.equals(house.startdate) : house.startdate != null) return false; - if (enddate != null ? !enddate.equals(house.enddate) : house.enddate != null) return false; - if (statstatus != null ? !statstatus.equals(house.statstatus) : house.statstatus != null) return false; - if (normdoc != null ? !normdoc.equals(house.normdoc) : house.normdoc != null) return false; - if (counter != null ? !counter.equals(house.counter) : house.counter != null) return false; - return cadnum != null ? cadnum.equals(house.cadnum) : house.cadnum == null; - - } - - @Override - public int hashCode() { - return Objects.hashCode(houseid); - } - - @Override - public String toString() { - return "House{" + - "aoguid='" + (aoguid!=null ? aoguid : "") + "'" + - ", houseid='" + (houseid!=null ? houseid : "") + "'" + - ", houseguid='" + (houseguid!=null ? houseguid : "") + "'" + - "}"; - } } \ No newline at end of file diff --git a/src/main/java/me/bearns/fias/helpers/UpdaterImpl.java b/src/main/java/me/bearns/fias/helpers/UpdaterImpl.java index 1552a5d..fe8225c 100644 --- a/src/main/java/me/bearns/fias/helpers/UpdaterImpl.java +++ b/src/main/java/me/bearns/fias/helpers/UpdaterImpl.java @@ -1,5 +1,6 @@ package me.bearns.fias.helpers; +import lombok.extern.slf4j.Slf4j; import me.bearns.fias.domain.FiasVersion; import me.bearns.fias.exceptions.CommonException; import me.bearns.fias.exceptions.DownloadException; @@ -26,6 +27,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipException; import java.util.zip.ZipFile; +@Slf4j @Component public class UpdaterImpl implements UpdateHelper { @@ -47,24 +49,32 @@ public class UpdaterImpl implements UpdateHelper { public void processUpdates(List updates, boolean reloadFlag) throws CommonException { if(reloadFlag) { + log.info("Clear tables"); + versions.deleteAll(); catalog.getAll().stream().map(Catalog.Item::getRepository).forEach(JpaRepository::deleteAll); + + log.debug("Clear tables completed"); } updates.sort((o1, o2) -> Math.toIntExact(o1.getVersionId() - o2.getVersionId())); Map> map = new HashMap<>(); + log.debug("Start downloading"); //start download updates.forEach(u -> map.put(u, fileService.download(reloadFlag ? u.getFiasCompleteXmlUrl() : u.getFiasDeltaXmlUrl()))); //strict order for (FiasVersion item : updates) { + log.debug("Apply version {}", item); + //set region filter final String regions = item.getRegions(); RegionFilter filter = null; if(regions != null) { + log.debug("Use region filter ({})", regions); filter = new RegionFilter(regions); } @@ -72,19 +82,23 @@ public class UpdaterImpl implements UpdateHelper { try { //wait for downloading file = map.get(item).get(); + log.debug("Download completed"); //process update processArchive(file, filter); + + log.debug("Archive process Ok"); + //apply this version versions.save(item); + log.debug("Save version {}", item); + } catch (InterruptedException e) { - e.printStackTrace(); - //todo log + log.error("InterruptedException while download file"); throw new DownloadException(e); } catch (ExecutionException e) { - e.printStackTrace(); - //todo log + log.error("ExecutionException while download file"); throw new DownloadException(e); } } @@ -94,24 +108,36 @@ public class UpdaterImpl implements UpdateHelper { try(final ZipFile zipFile = new ZipFile(file)){ + log.debug("Process Zip file"); + final Enumeration entries = zipFile.entries(); while (entries.hasMoreElements()){ final ZipEntry entry = entries.nextElement(); + final String name = entry.getName(); - final UnmarshallerParameters config = catalog.getByPrefix(entry.getName()); + log.debug("Find '{}' in zip", name); - if(config!=null) { //todo + final UnmarshallerParameters config = catalog.getByPrefix(name); + + if(config!=null) { try (InputStream is = zipFile.getInputStream(entry)) { + log.debug("Process '{}' in zip", name); unmarshaller.process(is, config, filter); } + } else { + log.debug("Skip '{}'", name); } } + log.debug("Completed Zip file"); + } catch (ZipException e) { + log.error("Common Zip exception"); throw new UnzipException(e); } catch (IOException e) { + log.error("IO Unzip Exception"); throw new UnzipException(e); } diff --git a/src/main/java/me/bearns/fias/service/DownloaderImpl.java b/src/main/java/me/bearns/fias/service/DownloaderImpl.java index babe797..3225d08 100644 --- a/src/main/java/me/bearns/fias/service/DownloaderImpl.java +++ b/src/main/java/me/bearns/fias/service/DownloaderImpl.java @@ -1,5 +1,6 @@ package me.bearns.fias.service; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import java.io.*; @@ -10,16 +11,19 @@ import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.concurrent.*; +@Slf4j @Component public class DownloaderImpl implements Downloader { private static final ExecutorService pool = Executors.newCachedThreadPool(); private static final String BASE_DIRECTORY = "/tmp/"; //todo path - private static final ConcurrentMap> locks = new ConcurrentHashMap(); + private static final ConcurrentMap> locks = new ConcurrentHashMap(); @Override public CompletableFuture download(String url) { + log.info("Request to download url {}", url); + CompletableFuture start = new CompletableFuture<>(); //syncronized @@ -36,6 +40,8 @@ public class DownloaderImpl implements Downloader { final URL conn; final Path path; + log.info("Start downloading {}", url); + try{ conn = new URL(url); path = Paths.get(BASE_DIRECTORY, conn.getFile()); @@ -52,14 +58,23 @@ public class DownloaderImpl implements Downloader { try (BufferedInputStream in = new BufferedInputStream(conn.openStream())) { - Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + final long size = Files.copy(in, path, StandardCopyOption.REPLACE_EXISTING); + + log.info("Download {} completed, {} bytes received", url, size); } catch (IOException e) { + + log.error("IOException while downloading url {}", url); // handle exception feature.obtrudeException(e); + return; } + } else { + log.warn("File {} already exist", file.getAbsoluteFile()); } + log.info("Downloaded url {} to file {}", url, file.getAbsoluteFile()); + feature.complete(file); }); diff --git a/src/main/java/me/bearns/fias/service/FiasUpdater.java b/src/main/java/me/bearns/fias/service/FiasUpdater.java index da2438e..41e239b 100644 --- a/src/main/java/me/bearns/fias/service/FiasUpdater.java +++ b/src/main/java/me/bearns/fias/service/FiasUpdater.java @@ -1,27 +1,16 @@ package me.bearns.fias.service; +import lombok.extern.slf4j.Slf4j; import me.bearns.fias.domain.FiasVersion; import me.bearns.fias.exceptions.CommonException; -import me.bearns.fias.exceptions.DownloadException; -import me.bearns.fias.exceptions.UnzipException; import me.bearns.fias.helpers.UpdateHelper; import me.bearns.fias.repository.FiasVersionRepository; -import me.bearns.fias.helpers.Catalog; -import me.bearns.fias.helpers.RegionFilter; -import me.bearns.fias.helpers.UnmarshallerParameters; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import java.io.*; import java.util.*; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.function.Predicate; -import java.util.zip.ZipEntry; -import java.util.zip.ZipException; -import java.util.zip.ZipFile; +@Slf4j @Service public class FiasUpdater implements Updater { @@ -37,6 +26,8 @@ public class FiasUpdater implements Updater { @Override public void update() throws CommonException { + log.info("Start update process"); + //Max value FiasVersion dbVersionObj = versions.findTopByOrderByVersionIdDesc(); @@ -47,34 +38,46 @@ public class FiasUpdater implements Updater { final long lastOnlineVersion = clientStub.getLastVersionId(); + log.info("Current version is {}, last online version {}", dbVersion, lastOnlineVersion); + if(lastOnlineVersion > dbVersion) { //need update final List updates = clientStub.getVersionsAfter(dbVersion); - //set region filters for updates - if(regions != null) updates.forEach(u -> u.setRegions(regions)); + log.debug("Need to apply updates {}", updates); if(updates != null && !updates.isEmpty()) { + //set region filters for updates + if(regions != null) updates.forEach(u -> u.setRegions(regions)); + helper.processUpdates(updates, false); + + log.info("Update completed"); } } + } else { + log.error("Can't get current version from db, nothing to update"); } - - System.out.println("update() Ok"); } @Override public void reload(Long... regions) throws CommonException { + log.info("Start reload process"); + final List lastVersion = clientStub.getLastVersion(); + log.debug("Reload version {}", lastVersion); + //exist and only one if(lastVersion != null && lastVersion.size() == 1) { //set region filter lastVersion.get(0).setRegions(regions); helper.processUpdates(lastVersion, true); + + log.info("Reload comleted"); } } diff --git a/src/main/java/me/bearns/fias/service/SOAPClient.java b/src/main/java/me/bearns/fias/service/SOAPClient.java index 41c43e6..66e9106 100644 --- a/src/main/java/me/bearns/fias/service/SOAPClient.java +++ b/src/main/java/me/bearns/fias/service/SOAPClient.java @@ -1,5 +1,6 @@ package me.bearns.fias.service; +import lombok.extern.slf4j.Slf4j; import me.bearns.fias.domain.FiasVersion; import org.springframework.stereotype.Component; import ru.nalog.fias.ArrayOfDownloadFileInfo; @@ -9,7 +10,6 @@ import ru.nalog.fias.IDownloadService_Service; import sun.net.www.protocol.http.HttpURLConnection; import javax.xml.ws.BindingProvider; -import javax.xml.ws.Service; import java.io.File; import java.io.InputStream; import java.net.URL; @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import static javax.xml.ws.BindingProvider.ENDPOINT_ADDRESS_PROPERTY; +@Slf4j @Component public class SOAPClient implements OnlineVersion { @@ -31,6 +32,8 @@ public class SOAPClient implements OnlineVersion { public SOAPClient() { + log.info("Init SOAP client"); + try (InputStream stream = getClass().getResourceAsStream(RESOURCE_WSDL)) { final File tempFile = File.createTempFile(RESOURCE_WSDL, "tmp"); @@ -40,17 +43,22 @@ public class SOAPClient implements OnlineVersion { if(Files.copy(stream, tempFile.toPath(), StandardCopyOption.REPLACE_EXISTING) > 0) { //copy some bytes ok + log.debug("WSDL saved to tmp file, try to init service"); //Init soap client IDownloadService_Service service = new IDownloadService_Service(tempFile.toURI().toURL()); downloadService = service.getBasicHttpBindingIDownloadService(); + log.debug("Service ok, enable redirect hook"); + setRedirect(downloadService); + + log.info("SOAP client init completed"); } }catch (Exception e) { - + log.error("Caught exception while init SOAP client"); } } @@ -141,7 +149,7 @@ public class SOAPClient implements OnlineVersion { } } } catch(final Exception e) { - e.printStackTrace(); + log.error("Endpoint redirect not set"); } } }