From 40f91ca410d0807155da6af93d70e447ab85eefe Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Mon, 18 Dec 2017 21:36:10 +0100
Subject: [PATCH 01/69] update bitcoinj

---
 build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sbt b/build.sbt
index 057388e..af3aa69 100644
--- a/build.sbt
+++ b/build.sbt
@@ -10,7 +10,7 @@ maintainer := "Bitcoinprivacy <info@bitcoinprivacy.net>"
 
 // additional libraries
 libraryDependencies ++= Seq(
-  "org.bitcoinj" % "bitcoinj-core" % "0.13.6",
+  "org.bitcoinj" % "bitcoinj-core" % "0.14.5",
   "org.xerial.snappy"%"snappy-java"%"1.1.2.4",
   "org.iq80.leveldb"%"leveldb"%"0.7",
   "org.fusesource.leveldbjni"%"leveldbjni-all"%"1.8",

From 20af5a3c2ea8fb680cf7ccb063aeb91b26b2c48c Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Mon, 18 Dec 2017 21:44:43 +0100
Subject: [PATCH 02/69] make network configurable (main, testnet, regtest)

---
 src/main/resources/reference.conf |  4 ++--
 src/main/scala/util/package.scala | 18 ++++++++++++++----
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index d0ceb0a..a0ea51c 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -17,5 +17,5 @@ host = "localhost"
 jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
 mdb.path = ${dataDir}"mdb/"
 mdb.transactionSize = 100000
-
-logger.scala.slick=ERROR
\ No newline at end of file
+network=main # accepts main, testnet, regtest
+logger.scala.slick=ERROR
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 286361c..9495d54 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
 package object util 
 {
   lazy val conf = ConfigFactory.load()
-  
+  lazy val networkMode = conf.getString("network")  
   lazy val closureTransactionSize = conf.getInt("closureTransactionSize")
   lazy val closureReadSize = conf.getInt("closureReadSize")
   lazy val populateTransactionSize = conf.getInt("populateTransactionSize")
@@ -28,12 +28,18 @@ package object util
   lazy val blockStoreFile = new java.io.File(conf.getString("levelDBFile"))
   lazy val lockFile = conf.getString("lockFile")
 
-  def params = MainNetParams.get
+  def params = 
+    if (networkMode == "main")
+      MainNetParams.get
+    else if (networkMode == "testnet")
+      TestNetParams.get()
+    else if (networkMode == "regtest")
+      RegTestParams.get()
+    else 
+      throw new Exception("Invalid parameter 'network'. Valid values 'main', 'testnet' or 'regtest'
 
   val log = Logger(LoggerFactory.getLogger("bge"))
 
-
-
   lazy val blockStore = new LevelDBBlockStore(new Context(params), blockStoreFile) //, factory)
   lazy val chain = new BlockChain(params, blockStore)
   lazy val peerGroup = new PeerGroup(params, chain)
@@ -44,6 +50,10 @@ package object util
    
   def startBitcoinJ: Unit = {
     log.info("starting peergroup")
+    
+    if (List("testnet", "regtest").contains(networkMode) {
+        peerGroup.connectToLocalHost()
+    }
     peerGroup.start
     peerGroup.addAddress(addr)
     peerGroup.waitForPeers(1).get();

From 65a1b0c4030b315dba2db247e69ffd2a80f3e6c8 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Mon, 18 Dec 2017 23:32:21 +0000
Subject: [PATCH 03/69] fix new configuration to get testnet bge

---
 src/main/resources/reference.conf |  2 +-
 src/main/scala/util/package.scala | 13 ++++---------
 2 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index a0ea51c..51e2bb4 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -17,5 +17,5 @@ host = "localhost"
 jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
 mdb.path = ${dataDir}"mdb/"
 mdb.transactionSize = 100000
-network=main # accepts main, testnet, regtest
+network=testnet # accepts main, testnet, regtest
 logger.scala.slick=ERROR
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 9495d54..07c70b2 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -5,7 +5,7 @@
 import com.typesafe.config.ConfigFactory
 import java.net.InetAddress
 import org.bitcoinj.core._
-import org.bitcoinj.params.MainNetParams
+import org.bitcoinj.params._
 import org.bitcoinj.store._
 import org.bitcoinj.utils.BlockFileLoader
 import scala.collection.mutable.ArrayBuffer
@@ -32,11 +32,9 @@ package object util
     if (networkMode == "main")
       MainNetParams.get
     else if (networkMode == "testnet")
-      TestNetParams.get()
-    else if (networkMode == "regtest")
-      RegTestParams.get()
-    else 
-      throw new Exception("Invalid parameter 'network'. Valid values 'main', 'testnet' or 'regtest'
+      TestNet3Params.get
+    else
+      RegTestParams.get
 
   val log = Logger(LoggerFactory.getLogger("bge"))
 
@@ -51,9 +49,6 @@ package object util
   def startBitcoinJ: Unit = {
     log.info("starting peergroup")
     
-    if (List("testnet", "regtest").contains(networkMode) {
-        peerGroup.connectToLocalHost()
-    }
     peerGroup.start
     peerGroup.addAddress(addr)
     peerGroup.waitForPeers(1).get();

From 6818c03678b699cc1e78d1d722c6417a223329a9 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 19 Dec 2017 09:30:55 +0000
Subject: [PATCH 04/69] use custom fileLoader for block reading in testnet3

---
 .../bge/core/TestNetBlockFileLoader.java      | 176 ++++++++++++++++++
 src/main/scala/util/package.scala             |   7 +-
 2 files changed, 182 insertions(+), 1 deletion(-)
 create mode 100644 src/main/scala/bge/core/TestNetBlockFileLoader.java

diff --git a/src/main/scala/bge/core/TestNetBlockFileLoader.java b/src/main/scala/bge/core/TestNetBlockFileLoader.java
new file mode 100644
index 0000000..b70d2fa
--- /dev/null
+++ b/src/main/scala/bge/core/TestNetBlockFileLoader.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright 2012 Matt Corallo.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package util;
+
+import org.bitcoinj.core.Block;
+import org.bitcoinj.core.NetworkParameters;
+import org.bitcoinj.core.ProtocolException;
+import org.bitcoinj.core.Utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.NoSuchElementException;
+
+/**
+ * <p>This class reads block files stored in the Bitcoin Core format. This is simply a way to concatenate
+ * blocks together. Importing block data with this tool can be a lot faster than syncing over the network, if you
+ * have the files available.</p>
+ * 
+ * <p>In order to comply with Iterator&lt;Block>, this class swallows a lot of IOExceptions, which may result in a few
+ * blocks being missed followed by a huge set of orphan blocks.</p>
+ * 
+ * <p>To blindly import all files which can be found in Bitcoin Core (version >= 0.8) datadir automatically,
+ * try this code fragment:<br>
+ * BlockFileLoader loader = new BlockFileLoader(BlockFileLoader.getReferenceClientBlockFileList());<br>
+ * for (Block block : loader) {<br>
+ * &nbsp;&nbsp;try { chain.add(block); } catch (Exception e) { }<br>
+ * }</p>
+ */
+public class TestNetBlockFileLoader implements Iterable<Block>, Iterator<Block> {
+    /**
+     * Gets the list of files which contain blocks from Bitcoin Core.
+     */
+    public static List<File> getReferenceClientBlockFileList() {
+        String defaultDataDir;
+        String OS = System.getProperty("os.name").toLowerCase();
+        if (OS.indexOf("win") >= 0) {
+            defaultDataDir = System.getenv("APPDATA") + "\\.bitcoin\\blocks\\";
+        } else if (OS.indexOf("mac") >= 0 || (OS.indexOf("darwin") >= 0)) {
+            defaultDataDir = System.getProperty("user.home") + "/Library/Application Support/Bitcoin/blocks/";
+        } else {
+            defaultDataDir = System.getProperty("user.home") + "/.bitcoin/testnet3/blocks/";
+        }
+        
+        List<File> list = new LinkedList<>();
+        for (int i = 0; true; i++) {
+            File file = new File(defaultDataDir + String.format(Locale.US, "blk%05d.dat", i));
+            if (!file.exists())
+                break;
+            list.add(file);
+        }
+        return list;
+    }
+    
+    private Iterator<File> fileIt;
+    private FileInputStream currentFileStream = null;
+    private Block nextBlock = null;
+    private NetworkParameters params;
+    
+    public TestNetBlockFileLoader(NetworkParameters params, List<File> files) {
+        fileIt = files.iterator();
+        this.params = params;
+    }
+    
+    @Override
+    public boolean hasNext() {
+        if (nextBlock == null)
+            loadNextBlock();
+        return nextBlock != null;
+    }
+
+    @Override
+    public Block next() throws NoSuchElementException {
+        if (!hasNext())
+            throw new NoSuchElementException();
+        Block next = nextBlock;
+        nextBlock = null;
+        return next;
+    }
+    
+    private void loadNextBlock() {
+        while (true) {
+            try {
+                if (!fileIt.hasNext() && (currentFileStream == null || currentFileStream.available() < 1))
+                    break;
+            } catch (IOException e) {
+                currentFileStream = null;
+                if (!fileIt.hasNext())
+                    break;
+            }
+            while (true) {
+                try {
+                    if (currentFileStream != null && currentFileStream.available() > 0)
+                        break;
+                } catch (IOException e1) {
+                    currentFileStream = null;
+                }
+                if (!fileIt.hasNext()) {
+                    nextBlock = null;
+                    currentFileStream = null;
+                    return;
+                }
+                try {
+                    currentFileStream = new FileInputStream(fileIt.next());
+                } catch (FileNotFoundException e) {
+                    currentFileStream = null;
+                }
+            }
+            try {
+                int nextChar = currentFileStream.read();
+                while (nextChar != -1) {
+                    if (nextChar != ((params.getPacketMagic() >>> 24) & 0xff)) {
+                        nextChar = currentFileStream.read();
+                        continue;
+                    }
+                    nextChar = currentFileStream.read();
+                    if (nextChar != ((params.getPacketMagic() >>> 16) & 0xff))
+                        continue;
+                    nextChar = currentFileStream.read();
+                    if (nextChar != ((params.getPacketMagic() >>> 8) & 0xff))
+                        continue;
+                    nextChar = currentFileStream.read();
+                    if (nextChar == (params.getPacketMagic() & 0xff))
+                        break;
+                }
+                byte[] bytes = new byte[4];
+                currentFileStream.read(bytes, 0, 4);
+                long size = Utils.readUint32BE(Utils.reverseBytes(bytes), 0);
+                // We allow larger than MAX_BLOCK_SIZE because test code uses this as well.
+                if (size > Block.MAX_BLOCK_SIZE*2 || size <= 0)
+                    continue;
+                bytes = new byte[(int) size];
+                currentFileStream.read(bytes, 0, (int) size);
+                try {
+                    nextBlock = params.getDefaultSerializer().makeBlock(bytes);
+                } catch (ProtocolException e) {
+                    nextBlock = null;
+                    continue;
+                }
+                break;
+            } catch (IOException e) {
+                currentFileStream = null;
+                continue;
+            }
+        }
+    }
+
+    @Override
+    public void remove() throws UnsupportedOperationException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Iterator<Block> iterator() {
+        return this;
+    }
+}
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 07c70b2..88a99bf 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -42,7 +42,12 @@ package object util
   lazy val chain = new BlockChain(params, blockStore)
   lazy val peerGroup = new PeerGroup(params, chain)
   lazy val loader = {
-    new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)
+    if (networkMode == "main")
+      new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)
+    else if (networkMode == "testnet")
+      new TestNetBlockFileLoader(params,TestNetBlockFileLoader.getReferenceClientBlockFileList)
+    else
+      throw new Exception(s"Not implemented FileLoader for $networkMode")
   }
   lazy val addr = new PeerAddress(InetAddress.getLocalHost(), params.getPort());
    

From bf1219ea914d81a02a5913dd65cf361db96eabaf Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Tue, 19 Dec 2017 10:31:55 +0100
Subject: [PATCH 05/69] revert default value of reference.conf and add try
 catch to stop populate whenever an error happens

---
 src/main/resources/reference.conf         |  2 +-
 src/main/scala/bge/core/BlockReader.scala | 22 +++++++++++++++-------
 2 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index 51e2bb4..a0ea51c 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -17,5 +17,5 @@ host = "localhost"
 jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
 mdb.path = ${dataDir}"mdb/"
 mdb.transactionSize = 100000
-network=testnet # accepts main, testnet, regtest
+network=main # accepts main, testnet, regtest
 logger.scala.slick=ERROR
diff --git a/src/main/scala/bge/core/BlockReader.scala b/src/main/scala/bge/core/BlockReader.scala
index a7c23e0..66da0ec 100644
--- a/src/main/scala/bge/core/BlockReader.scala
+++ b/src/main/scala/bge/core/BlockReader.scala
@@ -44,14 +44,22 @@ trait BlockReader extends BlockSource {
   }
 
   def process: Unit = {
-    for ((block, height) <- filteredBlockSource)
-    {
-      for (transaction <- transactionsInBlock(block)) {
-        saveTransaction(transaction, height)
-        transactionCounter +=1
+    try {
+      for ((block, height) <- filteredBlockSource)
+      {
+        for (transaction <- transactionsInBlock(block)) {
+          saveTransaction(transaction, height)
+          transactionCounter +=1
+        }
+        val blockHash = Hash(block.getHash.getBytes)
+        finishBlock(blockHash, block.getTransactions.size,getTxValue(block),block.getTimeSeconds,height)
+        if (height > 120000) throw new Exception("motherfuckers")
       }
-      val blockHash = Hash(block.getHash.getBytes)
-      finishBlock(blockHash, block.getTransactions.size,getTxValue(block),block.getTimeSeconds,height)
+    }
+    catch {
+      case e: Exception => 
+        log.error(e.getMessage)
+        log.warn("Try to recovery starting resume at these point")
     }
   }
 

From 829b042789270b251329adec316a6d034babafa8 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 19 Dec 2017 09:39:17 +0000
Subject: [PATCH 06/69] remove debug code

---
 src/main/scala/bge/core/BlockReader.scala | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/main/scala/bge/core/BlockReader.scala b/src/main/scala/bge/core/BlockReader.scala
index 66da0ec..b16ee2c 100644
--- a/src/main/scala/bge/core/BlockReader.scala
+++ b/src/main/scala/bge/core/BlockReader.scala
@@ -53,7 +53,6 @@ trait BlockReader extends BlockSource {
         }
         val blockHash = Hash(block.getHash.getBytes)
         finishBlock(blockHash, block.getTransactions.size,getTxValue(block),block.getTimeSeconds,height)
-        if (height > 120000) throw new Exception("motherfuckers")
       }
     }
     catch {

From a02c3a70a8c36cd397d1ccc4e57d697a3a6b3220 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 19 Dec 2017 14:09:47 +0000
Subject: [PATCH 07/69] fix compilation issues

---
 api/project/build.scala                       |   2 +-
 build.sbt                                     |   6 +-
 src/main/resources/reference.conf             |   2 +-
 src/main/scala/bge/core/BlockReader.scala     |  21 ++-
 .../bge/core/RegTestBlockFileLoader.java      | 176 ++++++++++++++++++
 .../bge/core/TestNetBlockFileLoader.java      | 176 ++++++++++++++++++
 src/main/scala/util/package.scala             |   9 +-
 7 files changed, 378 insertions(+), 14 deletions(-)
 create mode 100644 src/main/scala/bge/core/RegTestBlockFileLoader.java
 create mode 100644 src/main/scala/bge/core/TestNetBlockFileLoader.java

diff --git a/api/project/build.scala b/api/project/build.scala
index a21241b..ec2506e 100644
--- a/api/project/build.scala
+++ b/api/project/build.scala
@@ -11,7 +11,7 @@ import com.earldouglas.xwp.ContainerPlugin.autoImport._
 object BgeapiBuild extends Build {
   val Organization = "net.bitcoinprivacy"
   val Name = "bgeapi"
-  val Version = "3.3"
+  val Version = "3.3-SNAPSHOTj"
   val ScalaVersion = "2.11.8"
   val ScalatraVersion = "2.4.0"
 
diff --git a/build.sbt b/build.sbt
index af3aa69..e25754e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -2,7 +2,7 @@ organization := "net.bitcoinprivacy"
 
 name := "bge"
 
-version := "3.3"
+version := "3.3-SNAPSHOT"
 
 scalaVersion := "2.11.8"
 
@@ -10,7 +10,7 @@ maintainer := "Bitcoinprivacy <info@bitcoinprivacy.net>"
 
 // additional libraries
 libraryDependencies ++= Seq(
-  "org.bitcoinj" % "bitcoinj-core" % "0.14.5",
+  "org.bitcoinj" % "bitcoinj-core" % "0.15-SNAPSHOT",
   "org.xerial.snappy"%"snappy-java"%"1.1.2.4",
   "org.iq80.leveldb"%"leveldb"%"0.7",
   "org.fusesource.leveldbjni"%"leveldbjni-all"%"1.8",
@@ -34,7 +34,7 @@ enablePlugins(JavaAppPackaging)
 
 libraryDependencies ~= { _.map(_.exclude("org.slf4j", "slf4j-simple")) }
 
-resolvers += "Local Maven Repository" at "file:///"+Path.userHome.absolutePath+"/.m2/repository"
+resolvers += "Local Maven Repository" at "file:///root/.m2/repository"
 
 resolvers += "Fakod Snapshots" at "https://raw.github.com/FaKod/fakod-mvn-repo/master/snapshots"
 
diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index 51e2bb4..a0ea51c 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -17,5 +17,5 @@ host = "localhost"
 jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
 mdb.path = ${dataDir}"mdb/"
 mdb.transactionSize = 100000
-network=testnet # accepts main, testnet, regtest
+network=main # accepts main, testnet, regtest
 logger.scala.slick=ERROR
diff --git a/src/main/scala/bge/core/BlockReader.scala b/src/main/scala/bge/core/BlockReader.scala
index a7c23e0..b16ee2c 100644
--- a/src/main/scala/bge/core/BlockReader.scala
+++ b/src/main/scala/bge/core/BlockReader.scala
@@ -44,14 +44,21 @@ trait BlockReader extends BlockSource {
   }
 
   def process: Unit = {
-    for ((block, height) <- filteredBlockSource)
-    {
-      for (transaction <- transactionsInBlock(block)) {
-        saveTransaction(transaction, height)
-        transactionCounter +=1
+    try {
+      for ((block, height) <- filteredBlockSource)
+      {
+        for (transaction <- transactionsInBlock(block)) {
+          saveTransaction(transaction, height)
+          transactionCounter +=1
+        }
+        val blockHash = Hash(block.getHash.getBytes)
+        finishBlock(blockHash, block.getTransactions.size,getTxValue(block),block.getTimeSeconds,height)
       }
-      val blockHash = Hash(block.getHash.getBytes)
-      finishBlock(blockHash, block.getTransactions.size,getTxValue(block),block.getTimeSeconds,height)
+    }
+    catch {
+      case e: Exception => 
+        log.error(e.getMessage)
+        log.warn("Try to recovery starting resume at these point")
     }
   }
 
diff --git a/src/main/scala/bge/core/RegTestBlockFileLoader.java b/src/main/scala/bge/core/RegTestBlockFileLoader.java
new file mode 100644
index 0000000..21e6bbf
--- /dev/null
+++ b/src/main/scala/bge/core/RegTestBlockFileLoader.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright 2012 Matt Corallo.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package util;
+
+import org.bitcoinj.core.Block;
+import org.bitcoinj.core.NetworkParameters;
+import org.bitcoinj.core.ProtocolException;
+import org.bitcoinj.core.Utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.NoSuchElementException;
+
+/**
+ * <p>This class reads block files stored in the Bitcoin Core format. This is simply a way to concatenate
+ * blocks together. Importing block data with this tool can be a lot faster than syncing over the network, if you
+ * have the files available.</p>
+ * 
+ * <p>In order to comply with Iterator&lt;Block>, this class swallows a lot of IOExceptions, which may result in a few
+ * blocks being missed followed by a huge set of orphan blocks.</p>
+ * 
+ * <p>To blindly import all files which can be found in Bitcoin Core (version >= 0.8) datadir automatically,
+ * try this code fragment:<br>
+ * BlockFileLoader loader = new BlockFileLoader(BlockFileLoader.getReferenceClientBlockFileList());<br>
+ * for (Block block : loader) {<br>
+ * &nbsp;&nbsp;try { chain.add(block); } catch (Exception e) { }<br>
+ * }</p>
+ */
+public class RegTestBlockFileLoader implements Iterable<Block>, Iterator<Block> {
+    /**
+     * Gets the list of files which contain blocks from Bitcoin Core.
+     */
+    public static List<File> getReferenceClientBlockFileList() {
+        String defaultDataDir;
+        String OS = System.getProperty("os.name").toLowerCase();
+        if (OS.indexOf("win") >= 0) {
+            defaultDataDir = System.getenv("APPDATA") + "\\.bitcoin\\regtest\blocks\\";
+        } else if (OS.indexOf("mac") >= 0 || (OS.indexOf("darwin") >= 0)) {
+            defaultDataDir = System.getProperty("user.home") + "/Library/Application Support/Bitcoin/regtest/blocks/";
+        } else {
+            defaultDataDir = System.getProperty("user.home") + "/.bitcoin/regtest/blocks/";
+        }
+        
+        List<File> list = new LinkedList<>();
+        for (int i = 0; true; i++) {
+            File file = new File(defaultDataDir + String.format(Locale.US, "blk%05d.dat", i));
+            if (!file.exists())
+                break;
+            list.add(file);
+        }
+        return list;
+    }
+    
+    private Iterator<File> fileIt;
+    private FileInputStream currentFileStream = null;
+    private Block nextBlock = null;
+    private NetworkParameters params;
+    
+    public RegTestBlockFileLoader(NetworkParameters params, List<File> files) {
+        fileIt = files.iterator();
+        this.params = params;
+    }
+    
+    @Override
+    public boolean hasNext() {
+        if (nextBlock == null)
+            loadNextBlock();
+        return nextBlock != null;
+    }
+
+    @Override
+    public Block next() throws NoSuchElementException {
+        if (!hasNext())
+            throw new NoSuchElementException();
+        Block next = nextBlock;
+        nextBlock = null;
+        return next;
+    }
+    
+    private void loadNextBlock() {
+        while (true) {
+            try {
+                if (!fileIt.hasNext() && (currentFileStream == null || currentFileStream.available() < 1))
+                    break;
+            } catch (IOException e) {
+                currentFileStream = null;
+                if (!fileIt.hasNext())
+                    break;
+            }
+            while (true) {
+                try {
+                    if (currentFileStream != null && currentFileStream.available() > 0)
+                        break;
+                } catch (IOException e1) {
+                    currentFileStream = null;
+                }
+                if (!fileIt.hasNext()) {
+                    nextBlock = null;
+                    currentFileStream = null;
+                    return;
+                }
+                try {
+                    currentFileStream = new FileInputStream(fileIt.next());
+                } catch (FileNotFoundException e) {
+                    currentFileStream = null;
+                }
+            }
+            try {
+                int nextChar = currentFileStream.read();
+                while (nextChar != -1) {
+                    if (nextChar != ((params.getPacketMagic() >>> 24) & 0xff)) {
+                        nextChar = currentFileStream.read();
+                        continue;
+                    }
+                    nextChar = currentFileStream.read();
+                    if (nextChar != ((params.getPacketMagic() >>> 16) & 0xff))
+                        continue;
+                    nextChar = currentFileStream.read();
+                    if (nextChar != ((params.getPacketMagic() >>> 8) & 0xff))
+                        continue;
+                    nextChar = currentFileStream.read();
+                    if (nextChar == (params.getPacketMagic() & 0xff))
+                        break;
+                }
+                byte[] bytes = new byte[4];
+                currentFileStream.read(bytes, 0, 4);
+                long size = Utils.readUint32BE(Utils.reverseBytes(bytes), 0);
+                // We allow larger than MAX_BLOCK_SIZE because test code uses this as well.
+                if (size > Block.MAX_BLOCK_SIZE*2 || size <= 0)
+                    continue;
+                bytes = new byte[(int) size];
+                currentFileStream.read(bytes, 0, (int) size);
+                try {
+                    nextBlock = params.getDefaultSerializer().makeBlock(bytes);
+                } catch (ProtocolException e) {
+                    nextBlock = null;
+                    continue;
+                }
+                break;
+            } catch (IOException e) {
+                currentFileStream = null;
+                continue;
+            }
+        }
+    }
+
+    @Override
+    public void remove() throws UnsupportedOperationException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Iterator<Block> iterator() {
+        return this;
+    }
+}
diff --git a/src/main/scala/bge/core/TestNetBlockFileLoader.java b/src/main/scala/bge/core/TestNetBlockFileLoader.java
new file mode 100644
index 0000000..b70d2fa
--- /dev/null
+++ b/src/main/scala/bge/core/TestNetBlockFileLoader.java
@@ -0,0 +1,176 @@
+/*
+ * Copyright 2012 Matt Corallo.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package util;
+
+import org.bitcoinj.core.Block;
+import org.bitcoinj.core.NetworkParameters;
+import org.bitcoinj.core.ProtocolException;
+import org.bitcoinj.core.Utils;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.NoSuchElementException;
+
+/**
+ * <p>This class reads block files stored in the Bitcoin Core format. This is simply a way to concatenate
+ * blocks together. Importing block data with this tool can be a lot faster than syncing over the network, if you
+ * have the files available.</p>
+ * 
+ * <p>In order to comply with Iterator&lt;Block>, this class swallows a lot of IOExceptions, which may result in a few
+ * blocks being missed followed by a huge set of orphan blocks.</p>
+ * 
+ * <p>To blindly import all files which can be found in Bitcoin Core (version >= 0.8) datadir automatically,
+ * try this code fragment:<br>
+ * BlockFileLoader loader = new BlockFileLoader(BlockFileLoader.getReferenceClientBlockFileList());<br>
+ * for (Block block : loader) {<br>
+ * &nbsp;&nbsp;try { chain.add(block); } catch (Exception e) { }<br>
+ * }</p>
+ */
+public class TestNetBlockFileLoader implements Iterable<Block>, Iterator<Block> {
+    /**
+     * Gets the list of files which contain blocks from Bitcoin Core.
+     */
+    public static List<File> getReferenceClientBlockFileList() {
+        String defaultDataDir;
+        String OS = System.getProperty("os.name").toLowerCase();
+        if (OS.indexOf("win") >= 0) {
+            defaultDataDir = System.getenv("APPDATA") + "\\.bitcoin\\blocks\\";
+        } else if (OS.indexOf("mac") >= 0 || (OS.indexOf("darwin") >= 0)) {
+            defaultDataDir = System.getProperty("user.home") + "/Library/Application Support/Bitcoin/blocks/";
+        } else {
+            defaultDataDir = System.getProperty("user.home") + "/.bitcoin/testnet3/blocks/";
+        }
+        
+        List<File> list = new LinkedList<>();
+        for (int i = 0; true; i++) {
+            File file = new File(defaultDataDir + String.format(Locale.US, "blk%05d.dat", i));
+            if (!file.exists())
+                break;
+            list.add(file);
+        }
+        return list;
+    }
+    
+    private Iterator<File> fileIt;
+    private FileInputStream currentFileStream = null;
+    private Block nextBlock = null;
+    private NetworkParameters params;
+    
+    public TestNetBlockFileLoader(NetworkParameters params, List<File> files) {
+        fileIt = files.iterator();
+        this.params = params;
+    }
+    
+    @Override
+    public boolean hasNext() {
+        if (nextBlock == null)
+            loadNextBlock();
+        return nextBlock != null;
+    }
+
+    @Override
+    public Block next() throws NoSuchElementException {
+        if (!hasNext())
+            throw new NoSuchElementException();
+        Block next = nextBlock;
+        nextBlock = null;
+        return next;
+    }
+    
+    private void loadNextBlock() {
+        while (true) {
+            try {
+                if (!fileIt.hasNext() && (currentFileStream == null || currentFileStream.available() < 1))
+                    break;
+            } catch (IOException e) {
+                currentFileStream = null;
+                if (!fileIt.hasNext())
+                    break;
+            }
+            while (true) {
+                try {
+                    if (currentFileStream != null && currentFileStream.available() > 0)
+                        break;
+                } catch (IOException e1) {
+                    currentFileStream = null;
+                }
+                if (!fileIt.hasNext()) {
+                    nextBlock = null;
+                    currentFileStream = null;
+                    return;
+                }
+                try {
+                    currentFileStream = new FileInputStream(fileIt.next());
+                } catch (FileNotFoundException e) {
+                    currentFileStream = null;
+                }
+            }
+            try {
+                int nextChar = currentFileStream.read();
+                while (nextChar != -1) {
+                    if (nextChar != ((params.getPacketMagic() >>> 24) & 0xff)) {
+                        nextChar = currentFileStream.read();
+                        continue;
+                    }
+                    nextChar = currentFileStream.read();
+                    if (nextChar != ((params.getPacketMagic() >>> 16) & 0xff))
+                        continue;
+                    nextChar = currentFileStream.read();
+                    if (nextChar != ((params.getPacketMagic() >>> 8) & 0xff))
+                        continue;
+                    nextChar = currentFileStream.read();
+                    if (nextChar == (params.getPacketMagic() & 0xff))
+                        break;
+                }
+                byte[] bytes = new byte[4];
+                currentFileStream.read(bytes, 0, 4);
+                long size = Utils.readUint32BE(Utils.reverseBytes(bytes), 0);
+                // We allow larger than MAX_BLOCK_SIZE because test code uses this as well.
+                if (size > Block.MAX_BLOCK_SIZE*2 || size <= 0)
+                    continue;
+                bytes = new byte[(int) size];
+                currentFileStream.read(bytes, 0, (int) size);
+                try {
+                    nextBlock = params.getDefaultSerializer().makeBlock(bytes);
+                } catch (ProtocolException e) {
+                    nextBlock = null;
+                    continue;
+                }
+                break;
+            } catch (IOException e) {
+                currentFileStream = null;
+                continue;
+            }
+        }
+    }
+
+    @Override
+    public void remove() throws UnsupportedOperationException {
+        throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public Iterator<Block> iterator() {
+        return this;
+    }
+}
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 07c70b2..6f46423 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -42,9 +42,14 @@ package object util
   lazy val chain = new BlockChain(params, blockStore)
   lazy val peerGroup = new PeerGroup(params, chain)
   lazy val loader = {
-    new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)
+    if (networkMode == "main")
+      new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)
+    else if (networkMode == "testnet")
+      new TestNetBlockFileLoader(params,TestNetBlockFileLoader.getReferenceClientBlockFileList)
+    else
+      new RegTestBlockFileLoader(params,RegTestBlockFileLoader.getReferenceClientBlockFileList)
   }
-  lazy val addr = new PeerAddress(InetAddress.getLocalHost(), params.getPort());
+  lazy val addr = new PeerAddress(params, InetAddress.getLocalHost(), params.getPort());
    
   def startBitcoinJ: Unit = {
     log.info("starting peergroup")

From 5d44c51d4412e20fa8d9f153e3be43ff2d367b81 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 19 Dec 2017 16:41:19 +0000
Subject: [PATCH 08/69] fix compilation error

---
 .gitignore                                    | 3 ++-
 src/main/scala/bge/core/FastBlockReader.scala | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 589c9fb..37e83bc 100755
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,5 @@ neodb/
 .ensime
 .ensime_cache/
 src/main/resources/application.conf
-.history
\ No newline at end of file
+.history
+blockchain
diff --git a/src/main/scala/bge/core/FastBlockReader.scala b/src/main/scala/bge/core/FastBlockReader.scala
index 691af23..2d6ba7c 100644
--- a/src/main/scala/bge/core/FastBlockReader.scala
+++ b/src/main/scala/bge/core/FastBlockReader.scala
@@ -92,9 +92,9 @@ trait FastBlockReader extends BlockReader {
 
   def saveUnmatchedInputs: Unit =
   {
-    assert(outOfOrderInputMap.size == 0, "unmatched Inputs: " + outOfOrderInputMap.toList)
-    //for (((outpointTransactionHash, outpointIndex), transactionHash) <- outOfOrderInputMap)
-    //  insertInsertIntoList(Some(transactionHash), Some(outpointTransactionHash), None, Some(outpointIndex), None, None)
+    val unmatchedCount = outOfOrderInputMap.size
+    log.debug(outOfOrderInputMap.toList.toString)
+    assert(unmatchedCount == 0, "unmatched Inputs found")
   }
 
   def saveDataToDB: Unit =

From 8ca83831e701739879f65b752a984965f460facc Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 19 Dec 2017 20:18:25 +0000
Subject: [PATCH 09/69] enable use other networks than the main one

---
 api/project/build.scala                       |  3 ++-
 .../main/scala/api/MyScalatraServlet.scala    | 23 +++++++++++++++----
 api/src/main/scala/api/models/Address.scala   | 23 +++++++++++++++----
 3 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/api/project/build.scala b/api/project/build.scala
index ec2506e..4ea3764 100644
--- a/api/project/build.scala
+++ b/api/project/build.scala
@@ -11,7 +11,7 @@ import com.earldouglas.xwp.ContainerPlugin.autoImport._
 object BgeapiBuild extends Build {
   val Organization = "net.bitcoinprivacy"
   val Name = "bgeapi"
-  val Version = "3.3-SNAPSHOTj"
+  val Version = "3.3-SNAPSHOT"
   val ScalaVersion = "2.11.8"
   val ScalatraVersion = "2.4.0"
 
@@ -24,6 +24,7 @@ object BgeapiBuild extends Build {
       version := Version,
       scalaVersion := ScalaVersion,
       resolvers += Classpaths.typesafeReleases,
+      resolvers += "Local Maven Repository" at "file:///root/.m2/repository",
       resolvers += "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases",
       libraryDependencies ++= Seq(
         "net.bitcoinprivacy" %% "bge" % Version,
diff --git a/api/src/main/scala/api/MyScalatraServlet.scala b/api/src/main/scala/api/MyScalatraServlet.scala
index 42a8e9f..eb0255e 100644
--- a/api/src/main/scala/api/MyScalatraServlet.scala
+++ b/api/src/main/scala/api/MyScalatraServlet.scala
@@ -9,11 +9,26 @@ import org.json4s.{DefaultFormats, Formats}
 import org.scalatra.json._
 import net.bitcoinprivacy.bge.models._
 import org.bitcoinj.core.{Address => BitcoinJAddress}
-import org.bitcoinj.params.MainNetParams
+import org.bitcoinj.params._
 import org.scalatra.CorsSupport
+import com.typesafe.config.ConfigFactory
 
 class MyScalatraServlet extends BgeapiStack with db.BitcoinDB with JacksonJsonSupport with CorsSupport {
 
+  lazy val conf = ConfigFactory.load()
+  lazy val network = conf.getString("network")
+
+  lazy val networkParams = network match {
+    case "main" =>
+      MainNetParams.get
+    case "regtest" =>
+      RegTestParams.get
+    case "testnet" =>
+      TestNet3Params.get
+    case _ =>
+      throw new Exception("Unknow params for network"+network)
+					    }
+					    
   // Sets up automatic case class to JSON output serialization, required by
   // the JValueResult trait.
   protected implicit lazy val jsonFormats: Formats = DefaultFormats
@@ -144,13 +159,13 @@ class MyScalatraServlet extends BgeapiStack with db.BitcoinDB with JacksonJsonSu
   def hexAddress(stringAddress: String): String = {
     val arrayAddress = stringAddress.split(",")
     if (arrayAddress.length == 1) {
-      val address = new BitcoinJAddress(MainNetParams.get, stringAddress)
-                                       (if(address.isP2SHAddress) "05" else "00")+valueOf(address.getHash160)
+      val address = new BitcoinJAddress(networkParams, stringAddress)
+      (if(address.isP2SHAddress) "05" else "00")+valueOf(address.getHash160)
     }
     else{
       "0" + arrayAddress.length + 
         (for (i <- 0 until arrayAddress.length) 
-         yield  valueOf(new BitcoinJAddress(MainNetParams.get, arrayAddress(i)).getHash160) ).mkString("")
+         yield  valueOf(new BitcoinJAddress(networkParams, arrayAddress(i)).getHash160) ).mkString("")
     }
   }
 
diff --git a/api/src/main/scala/api/models/Address.scala b/api/src/main/scala/api/models/Address.scala
index e52f651..928afaa 100644
--- a/api/src/main/scala/api/models/Address.scala
+++ b/api/src/main/scala/api/models/Address.scala
@@ -2,15 +2,28 @@ package net.bitcoinprivacy.bge.models
 
 import db._
 import org.bitcoinj.core.{Address => Add}
-import org.bitcoinj.params.MainNetParams
+import org.bitcoinj.params._
 import scala.slick.driver.PostgresDriver.simple._
-
+import com.typesafe.config.ConfigFactory
 
 case class Address(address: String, balance: Long)
 case class AddressesSummary(count: Int, sum: Long)
 
 object Address extends db.BitcoinDB {
-
+  lazy val conf = ConfigFactory.load()
+  lazy val network = conf.getString("network")
+
+  lazy val params = network match {
+    case "main" =>
+      MainNetParams.get
+    case "regtest" =>
+      RegTestParams.get
+    case "testnet" =>
+      TestNet3Params.get
+    case _ =>
+      throw new Exception(s"Unknow params for network $network")
+  }
+					    
   def getWallet(hash: Array[Byte], from: Int, until: Int) = DB withSession { implicit session =>
 
     val repOpt = addresses.filter(_.hash === hash).map(_.representant).firstOption
@@ -82,9 +95,9 @@ object Address extends db.BitcoinDB {
 
   def hashToAddress(hash: Array[Byte]): String = hash.length match {
 
-    case 20 => new Add(MainNetParams.get,0,hash).toString
+    case 20 => new Add(params,0,hash).toString
 
-    case 21 => new Add(MainNetParams.get,hash.head.toInt,hash.tail).toString
+    case 21 => new Add(params,hash.head.toInt,hash.tail).toString
 
     case 0 => "No decodable address found"
 

From 7d7630b22432024aff0f8ddebfeee51f5d19e077 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 19 Dec 2017 21:19:45 +0000
Subject: [PATCH 10/69] use address prefix depending on network configuration

---
 .../main/scala/api/MyScalatraServlet.scala    | 20 ++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/api/src/main/scala/api/MyScalatraServlet.scala b/api/src/main/scala/api/MyScalatraServlet.scala
index eb0255e..af2f9de 100644
--- a/api/src/main/scala/api/MyScalatraServlet.scala
+++ b/api/src/main/scala/api/MyScalatraServlet.scala
@@ -27,9 +27,23 @@ class MyScalatraServlet extends BgeapiStack with db.BitcoinDB with JacksonJsonSu
       TestNet3Params.get
     case _ =>
       throw new Exception("Unknow params for network"+network)
-					    }
+
+  }
+
+  case class AddressPrefixes(p2pAddress: String, normalAddress: String)
+  lazy val prefix = network match {
+    case "main" =>
+      AddressPrefixes("05", "00")
+    case "regtest" =>
+      AddressPrefixes("C4", "6F")
+    case "testnet" =>
+      AddressPrefixes("C4", "6F")
+    case _ =>
+      throw new Exception("Unknow params for network"+network)
+
+  }
 					    
-  // Sets up automatic case class to JSON output serialization, required by
+  // Sets  up automatic case class to JSON output serialization, required by
   // the JValueResult trait.
   protected implicit lazy val jsonFormats: Formats = DefaultFormats
 
@@ -160,7 +174,7 @@ class MyScalatraServlet extends BgeapiStack with db.BitcoinDB with JacksonJsonSu
     val arrayAddress = stringAddress.split(",")
     if (arrayAddress.length == 1) {
       val address = new BitcoinJAddress(networkParams, stringAddress)
-      (if(address.isP2SHAddress) "05" else "00")+valueOf(address.getHash160)
+      (if(address.isP2SHAddress) prefix.p2pAddress else prefix.normalAddress)+valueOf(address.getHash160)
     }
     else{
       "0" + arrayAddress.length + 

From 10951e3b0ea2d35da5e126e8c0edd1ff7194f4b0 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Thu, 21 Dec 2017 12:01:24 +0000
Subject: [PATCH 11/69] catch address parse errors

---
 api/src/main/scala/api/models/Address.scala   | 9 +++++----
 src/main/scala/bge/core/FastBlockReader.scala | 4 +++-
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/api/src/main/scala/api/models/Address.scala b/api/src/main/scala/api/models/Address.scala
index 928afaa..5594c19 100644
--- a/api/src/main/scala/api/models/Address.scala
+++ b/api/src/main/scala/api/models/Address.scala
@@ -93,7 +93,7 @@ object Address extends db.BitcoinDB {
 
   }
 
-  def hashToAddress(hash: Array[Byte]): String = hash.length match {
+  def hashToAddress(hash: Array[Byte]): String = try {hash.length match {
 
     case 20 => new Add(params,0,hash).toString
 
@@ -102,11 +102,12 @@ object Address extends db.BitcoinDB {
     case 0 => "No decodable address found"
 
     case x if (x%20==1) =>
-
+      try {    
       (for (i <- 1 to hash.length-20 by 20)
        yield hashToAddress(hash.slice(i,i+20)) ).mkString(",")
-
+      } catch { case e:Exception => hash.toString }
     case _  => hash.length + " undefined"
-  }
+    }} catch { case _ => "Bitcoinj failed decoding address" }
+  
 
 }
diff --git a/src/main/scala/bge/core/FastBlockReader.scala b/src/main/scala/bge/core/FastBlockReader.scala
index 2d6ba7c..eaf6930 100644
--- a/src/main/scala/bge/core/FastBlockReader.scala
+++ b/src/main/scala/bge/core/FastBlockReader.scala
@@ -93,7 +93,9 @@ trait FastBlockReader extends BlockReader {
   def saveUnmatchedInputs: Unit =
   {
     val unmatchedCount = outOfOrderInputMap.size
-    log.debug(outOfOrderInputMap.toList.toString)
+    for (input <- outOfOrderInputMap){
+    	log.error(input.toString)
+    }
     assert(unmatchedCount == 0, "unmatched Inputs found")
   }
 

From 5d21c59375a7897d86f36f0568584fa6483bea7b Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Thu, 21 Dec 2017 13:52:16 +0000
Subject: [PATCH 12/69] use correct address version depending on the network

---
 api/src/main/scala/api/models/Address.scala |  4 ++--
 src/main/scala/bge/core/BlockReader.scala   | 18 +++---------------
 2 files changed, 5 insertions(+), 17 deletions(-)

diff --git a/api/src/main/scala/api/models/Address.scala b/api/src/main/scala/api/models/Address.scala
index 5594c19..a6cd13e 100644
--- a/api/src/main/scala/api/models/Address.scala
+++ b/api/src/main/scala/api/models/Address.scala
@@ -95,9 +95,9 @@ object Address extends db.BitcoinDB {
 
   def hashToAddress(hash: Array[Byte]): String = try {hash.length match {
 
-    case 20 => new Add(params,0,hash).toString
+    case 20 => new Add(params,params.getAddressHeader,hash).toString
 
-    case 21 => new Add(params,hash.head.toInt,hash.tail).toString
+    case 21 => new Add(params,params.getAddressHeader,hash.tail).toString
 
     case 0 => "No decodable address found"
 
diff --git a/src/main/scala/bge/core/BlockReader.scala b/src/main/scala/bge/core/BlockReader.scala
index b16ee2c..b05963f 100644
--- a/src/main/scala/bge/core/BlockReader.scala
+++ b/src/main/scala/bge/core/BlockReader.scala
@@ -44,9 +44,7 @@ trait BlockReader extends BlockSource {
   }
 
   def process: Unit = {
-    try {
-      for ((block, height) <- filteredBlockSource)
-      {
+      for ((block, height) <- filteredBlockSource) {
         for (transaction <- transactionsInBlock(block)) {
           saveTransaction(transaction, height)
           transactionCounter +=1
@@ -54,24 +52,14 @@ trait BlockReader extends BlockSource {
         val blockHash = Hash(block.getHash.getBytes)
         finishBlock(blockHash, block.getTransactions.size,getTxValue(block),block.getTimeSeconds,height)
       }
-    }
-    catch {
-      case e: Exception => 
-        log.error(e.getMessage)
-        log.warn("Try to recovery starting resume at these point")
-    }
   }
 
   def blockFilter(b: Block) = {
     val blockHash = Hash(b.getHash.getBytes)
-    !(savedBlockSet contains blockHash)
+    val exists = (savedBlockSet contains blockHash)
+    !exists 
   }
 
-  // def withoutDuplicates(b: Block, t: Transaction): Boolean =
-  //   !(Hash(b.getHash.getBytes) == Hash("00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec") &&
-  //     Hash(t.getHash.getBytes) == Hash("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599")) && the other transaction
- //  this isn't necessary anymore, because we simply update the LMDB with the same tx. So technically, we are missing those two duplicate tx in our utxo DB
-
   lazy val filteredBlockSource =
     blockSource withFilter (p=>blockFilter(p._1))
 

From 000fe645edbc72ac19cad1fdde390900b6a4ff12 Mon Sep 17 00:00:00 2001
From: Jorge <jorgemartinezpizarro@gmail.com>
Date: Thu, 28 Dec 2017 23:24:53 +0000
Subject: [PATCH 13/69] add parameters to configure bge

---
 src/main/resources/reference.conf                     |  3 +++
 .../scala/bge/core/BitcoinDRawFileBlockSource.scala   |  8 +++++++-
 src/main/scala/bge/core/PeerSource.scala              |  2 +-
 src/main/scala/util/package.scala                     | 11 +++++++++--
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index a0ea51c..37321c5 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -19,3 +19,6 @@ mdb.path = ${dataDir}"mdb/"
 mdb.transactionSize = 100000
 network=main # accepts main, testnet, regtest
 logger.scala.slick=ERROR
+resumeBlockSize=500 # depending on the available ram
+bitcoin.ip=localhost
+bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
diff --git a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
index 281c103..1b98ae8 100644
--- a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
+++ b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
@@ -12,7 +12,13 @@ trait BitcoinDRawFileBlockSource extends BlockSource
 
     startBitcoinJ
     log.info("starting")
-    val almostCurrentChain = getCurrentLongestChainFromBlockCount
+    val almostCurrentChain = if (maxPopulate == 0)
+	CurrentLongestChainFromBlockCount
+    else if (maxPopulate > 0)
+        CurrentLongestChainFromBlockCount.take(maxPopulate)
+    else
+	throw new Exception(s"invalid parameter bitcoin.maxPopulate = '$maxPopulate'")
+
     val blockMap = almostCurrentChain toMap
 
     for {
diff --git a/src/main/scala/bge/core/PeerSource.scala b/src/main/scala/bge/core/PeerSource.scala
index e24c358..e44e214 100644
--- a/src/main/scala/bge/core/PeerSource.scala
+++ b/src/main/scala/bge/core/PeerSource.scala
@@ -5,7 +5,7 @@ import util._
 
 trait PeerSource extends BlockSource {
   
-  lazy val truncated = getCurrentLongestChainFromBlockCount take 100 // take 100 so the changes don't get too big for memory
+  lazy val truncated = getCurrentLongestChainFromBlockCount take resumeBlockSize 
   
   override def blockSource =
   {
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index ee51aa9..7eaf420 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -24,10 +24,17 @@ package object util
   lazy val dustLimit = conf.getLong("dustLimit")
   lazy val dataDir = conf.getString( "dataDir")
   lazy val richlistSize = conf.getInt("richlistSize")
-
+  lazy val resumeBlockSize = conf.getInt("resumeBlockSize")
   lazy val blockStoreFile = new java.io.File(conf.getString("levelDBFile"))
   lazy val lockFile = conf.getString("lockFile")
+  lazy val internetAddress = bitcoinAddress = conf.getString("bitcoin.ip") match {
+    case "localhost" => 
+      InetAddress.getLocalHost()
+    case e: String =>
+      InetAddress.getByName(e)
+  lazy val maxPopulate = conf.getString("bitcoin.maxPopulate")
 
+  }
   def params = 
     if (networkMode == "main")
       MainNetParams.get
@@ -51,7 +58,7 @@ package object util
     else
       throw new Exception(s"Not implemented FileLoader for $networkMode")
   }
-  lazy val addr = new PeerAddress(params, InetAddress.getLocalHost(), params.getPort());
+  lazy val addr = new PeerAddress(params, internetAddress, params.getPort());
    
   def startBitcoinJ: Unit = {
     log.info("starting peergroup")

From 3698918fee78dcde3fc499dd3553e23f2b215800 Mon Sep 17 00:00:00 2001
From: Jorge <jorgemartinezpizarro@gmail.com>
Date: Thu, 28 Dec 2017 23:34:14 +0000
Subject: [PATCH 14/69] move magic numbers into configuration

---
 src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala | 4 ++--
 src/main/scala/util/package.scala                        | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
index 1b98ae8..9ace263 100644
--- a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
+++ b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
@@ -13,9 +13,9 @@ trait BitcoinDRawFileBlockSource extends BlockSource
     startBitcoinJ
     log.info("starting")
     val almostCurrentChain = if (maxPopulate == 0)
-	CurrentLongestChainFromBlockCount
+	getCurrentLongestChainFromBlockCount
     else if (maxPopulate > 0)
-        CurrentLongestChainFromBlockCount.take(maxPopulate)
+        getCurrentLongestChainFromBlockCount.take(maxPopulate)
     else
 	throw new Exception(s"invalid parameter bitcoin.maxPopulate = '$maxPopulate'")
 
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 7eaf420..b48fccf 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -27,14 +27,14 @@ package object util
   lazy val resumeBlockSize = conf.getInt("resumeBlockSize")
   lazy val blockStoreFile = new java.io.File(conf.getString("levelDBFile"))
   lazy val lockFile = conf.getString("lockFile")
-  lazy val internetAddress = bitcoinAddress = conf.getString("bitcoin.ip") match {
+  lazy val internetAddress = conf.getString("bitcoin.ip") match {
     case "localhost" => 
       InetAddress.getLocalHost()
     case e: String =>
       InetAddress.getByName(e)
-  lazy val maxPopulate = conf.getString("bitcoin.maxPopulate")
-
   }
+  lazy val maxPopulate = conf.getInt("bitcoin.maxPopulate")
+
   def params = 
     if (networkMode == "main")
       MainNetParams.get

From f9b1cbd719b9542c776de1e9e998a3e935b27ba1 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 4 Jan 2018 09:37:17 +0100
Subject: [PATCH 15/69] add config params to stats endpoint

---
 api/src/main/scala/api/models/Stats.scala | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/api/src/main/scala/api/models/Stats.scala b/api/src/main/scala/api/models/Stats.scala
index b4ae064..be8e2cb 100644
--- a/api/src/main/scala/api/models/Stats.scala
+++ b/api/src/main/scala/api/models/Stats.scala
@@ -1,19 +1,23 @@
 package net.bitcoinprivacy.bge.models
 
 import scala.slick.driver.PostgresDriver.simple._
+import com.typesafe.config.ConfigFactory
 
-
-case class Stats(block_height:Int, total_bitcoins_in_addresses:Long, total_transactions:Long, total_addresses:Long, total_closures:Long, total_addresses_with_balance:Long, total_closures_with_balance:Long, total_addresses_no_dust:Long, total_closures_no_dust:Long, gini_closure:Double,gini_address:Double, tstamp:Long)
+case class Stats(block_height:Int, total_bitcoins_in_addresses:Long, total_transactions:Long, total_addresses:Long, total_closures:Long, total_addresses_with_balance:Long, total_closures_with_balance:Long, total_addresses_no_dust:Long, total_closures_no_dust:Long, gini_closure:Double,gini_address:Double, tstamp:Long, network:String, dust: Int)
 
 
 object Stats extends db.BitcoinDB
-{ 
+{
+  lazy val conf = ConfigFactory.load()
+  lazy val dustLimit = conf.getInt("dustLimit")
+  lazy val network= conf.getString("network")
+
   def getStats =
     DB withSession { implicit session =>
       val o = stats.sortBy(_.block_height.desc).firstOption
 
       for (p <- o)
-      yield Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12)
+      yield Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12, network, dustLimit)
     }
 
   def getAllStats =
@@ -22,7 +26,7 @@ object Stats extends db.BitcoinDB
       val p1 = stats.sortBy(_.block_height.desc).run.toList
 
       p1 map {
-        p => Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12)
+        p => Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12, network, dustLimit)
       }
     }
 }

From 854e88fe01a37552dba35aaf719bb18f1b3df4e1 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 5 Jan 2018 12:35:14 +0100
Subject: [PATCH 16/69] remove duplicated index

---
 src/main/scala/db/BitcoinDB.scala | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index e3e070d..267afc5 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -1,4 +1,5 @@
 // this has all the database stuff.
+
 package db
 
 import slick.driver.PostgresDriver.simple._
@@ -110,8 +111,7 @@ trait BitcoinDB {
       // '\x' is not an address
       Q.updateNA("""insert into balances select address, sum(value) as balance from utxo where address != '\x' group by address;""").execute
 
-      (Q.u + "create index addresses_balance on balances(address)").execute
-      (Q.u + "create index balance on balances(balance)").execute
+     (Q.u + "create index balance on balances(balance)").execute
 
       Q.updateNA("""
          insert into closure_balances
@@ -122,8 +122,7 @@ trait BitcoinDB {
          group by address;
       """).execute
 
-      (Q.u + "create index addresses_balance_2 on closure_balances(address)").execute
-      (Q.u + "create index balance_2 on closure_balances(balance)").execute
+       (Q.u + "create index balance_2 on closure_balances(balance)").execute
 
       log.info("Balances created in %s s" format (System.currentTimeMillis - clock) / 1000)
     }

From 0b79271be487b0c376162daada14aa37c28d1d52 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 5 Jan 2018 12:49:38 +0100
Subject: [PATCH 17/69] add update balance limit in conf

---
 src/main/resources/reference.conf | 1 +
 src/main/scala/bge/Explorer.scala | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index 37321c5..16416f2 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -22,3 +22,4 @@ logger.scala.slick=ERROR
 resumeBlockSize=500 # depending on the available ram
 bitcoin.ip=localhost
 bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
+balanceUpdateLimit=50000 # for balance changes beyond this size, create new balance tables
diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 8445268..1b40a67 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -171,7 +171,7 @@ object Explorer extends App with db.BitcoinDB {
     
     log.info(changedAddresses.size + " addresses changed balance")
 
-    if (changedAddresses.size < 38749 )
+    if (changedAddresses.size < balanceUpdateLimit )
     {
       updateBalanceTables(changedAddresses, changedReps)
       insertRichestAddresses

From 01785d6b39b7e72c4afa8fa055332266e95727e2 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 5 Jan 2018 13:10:31 +0100
Subject: [PATCH 18/69] have you saved the files? No

---
 src/main/scala/util/package.scala | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index b48fccf..df621ad 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -25,6 +25,7 @@ package object util
   lazy val dataDir = conf.getString( "dataDir")
   lazy val richlistSize = conf.getInt("richlistSize")
   lazy val resumeBlockSize = conf.getInt("resumeBlockSize")
+  lazy val balanceUpdateLimit = conf.getInt("balanceUpdateLimit")
   lazy val blockStoreFile = new java.io.File(conf.getString("levelDBFile"))
   lazy val lockFile = conf.getString("lockFile")
   lazy val internetAddress = conf.getString("bitcoin.ip") match {

From 7e90de2bc88b4e193de74dec90d15e50b11765bd Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 5 Jan 2018 15:11:24 +0100
Subject: [PATCH 19/69] change rollBack strategy to potentially allow for
 rolling back multiple blocks

---
 src/main/scala/bge/Explorer.scala | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 1b40a67..4733ccf 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -116,19 +116,19 @@ object Explorer extends App with db.BitcoinDB {
     resumeStats(read.changedAddresses, closure.changedReps, closure.addedAds, closure.addedReps)
   }
 
+  def rollBackToLastStatIfNecessary: Unit =
+    for (block <- getWrongBlock){
+      rollBack(block)
+      rollBackToLastStatIfNecessary
+    }
+
   def iterateResume = {
     // Seq("bitcoind","-daemon").run
     
     if (!peerGroup.isRunning) startBitcoinJ
 
-    // if there are more stats than blocks we could delete it
-    for (block <- getWrongBlock){
+    rollBackToLastStatIfNecessary
       
-      rollBack(block)
-      populateStats 
-      assert(getWrongBlock == None, "The database is inconsistent. See logs for details.")
-    }
-
     while (new java.io.File(lockFile).exists)
     {
       if (blockCount > chain.getBestChainHeight)
@@ -164,7 +164,8 @@ object Explorer extends App with db.BitcoinDB {
     if (sameCount && sameValue && rightValue && rightBlock) None
     else if (utxosMaxHeight > bc -1) Some(utxosMaxHeight)
     else if (bc - 1 > lch) Some(bc-1)
-    else throw new Exception("This should not have happened. See logs for details.")
+    else Some(lch)
+      //throw new Exception("This should not have happened. See logs for details.")
   }
 
   def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int)  = {

From dfd849906201261c785715040fb3567c463a9873 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Wed, 10 Jan 2018 21:50:54 +0100
Subject: [PATCH 20/69] fix error obtaining address from hash

---
 api/src/main/scala/api/models/Address.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/src/main/scala/api/models/Address.scala b/api/src/main/scala/api/models/Address.scala
index a6cd13e..f925887 100644
--- a/api/src/main/scala/api/models/Address.scala
+++ b/api/src/main/scala/api/models/Address.scala
@@ -97,7 +97,7 @@ object Address extends db.BitcoinDB {
 
     case 20 => new Add(params,params.getAddressHeader,hash).toString
 
-    case 21 => new Add(params,params.getAddressHeader,hash.tail).toString
+    case 21 => new Add(params,hash.head.toInt,hash.tail).toString
 
     case 0 => "No decodable address found"
 

From 3e3927249ef6f7afb6269cfe8ea27e96dbdee4e6 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 22 Mar 2018 14:02:21 +0100
Subject: [PATCH 21/69] add --newstats flag to resume command

---
 src/main/scala/bge/Explorer.scala | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 4733ccf..03f4f24 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -17,7 +17,7 @@ object Explorer extends App with db.BitcoinDB {
       if (!statsDone || rest.headOption == Some("--force"))
         populate
       Seq("touch",lockFile).!
-      iterateResume
+      iterateResume()
 
     case "populate"::rest             =>
 
@@ -26,7 +26,7 @@ object Explorer extends App with db.BitcoinDB {
     case "resume"::rest =>
 
       Seq("touch",lockFile).!
-      iterateResume
+      iterateResume(rest.headOption==Some("--newstats"))
 
     case "stop"::rest =>
 
@@ -45,7 +45,7 @@ object Explorer extends App with db.BitcoinDB {
          start [--force]: populate if necessary or --force, then resume
          stop: ask bge gently to stop at the end of the current iteration. Note that thís can take a few days if you are in the populate phase.
          populate: create the database movements with movements and closures.  Deletes any existing data.
-         resume: update the database generated by populate with new incoming data. Works only if populate is already done.
+         resume [--newstats]: update the database generated by populate with new incoming data. Works only if populate is already done. The newstats flag forces a regeneration of balance, richlist and stats tables.
          info
       """)
   }
@@ -122,13 +122,14 @@ object Explorer extends App with db.BitcoinDB {
       rollBackToLastStatIfNecessary
     }
 
-  def iterateResume = {
+  def iterateResume(newstats: Boolean = false) = {
     // Seq("bitcoind","-daemon").run
     
     if (!peerGroup.isRunning) startBitcoinJ
 
     rollBackToLastStatIfNecessary
-      
+    if (newstats) populateStats
+          
     while (new java.io.File(lockFile).exists)
     {
       if (blockCount > chain.getBestChainHeight)

From 889c9cd4965b4e8e8c498f7343c4521581ef2e7b Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 23 Mar 2018 11:28:34 +0100
Subject: [PATCH 22/69] update everything to scala 2.12.4 and sbt 1.1.0

---
 build.sbt                                                | 8 ++++----
 project/build.properties                                 | 2 +-
 project/plugins.sbt                                      | 4 ++--
 src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala | 2 +-
 src/main/scala/util/LmdbMap.scala                        | 2 +-
 5 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/build.sbt b/build.sbt
index e25754e..c09d444 100644
--- a/build.sbt
+++ b/build.sbt
@@ -4,7 +4,7 @@ name := "bge"
 
 version := "3.3-SNAPSHOT"
 
-scalaVersion := "2.11.8"
+scalaVersion := "2.12.4"
 
 maintainer := "Bitcoinprivacy <info@bitcoinprivacy.net>"
 
@@ -17,10 +17,10 @@ libraryDependencies ++= Seq(
   "org.postgresql" % "postgresql" % "9.4-1200-jdbc41",
   "com.typesafe.slick" %% "slick" % "2.1.0",
   "com.typesafe" % "config" % "1.2.1",
-  "org.scalacheck" %% "scalacheck" % "1.12.4" % "test",
-  "org.scalatest" %% "scalatest" % "2.1.5" % "test",
+  "org.scalacheck" %% "scalacheck" % "1.13.5" % "test",
+  "org.scalatest" %% "scalatest" % "3.2.0-SNAP10" % "test",
   "org.deephacks.lmdbjni" % "lmdbjni" % "0.4.6",
-  "com.typesafe.scala-logging" %% "scala-logging" % "3.4.0",
+  "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0",
   "ch.qos.logback" %  "logback-classic" % "1.1.7",
   ////////////////////////////////////////////////////////////////////
   // select one of the following depending on your architecture
diff --git a/project/build.properties b/project/build.properties
index 59e7c05..66fe511 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -1 +1 @@
-sbt.version=0.13.11
\ No newline at end of file
+sbt.version=1.1.0
\ No newline at end of file
diff --git a/project/plugins.sbt b/project/plugins.sbt
index 796830d..8201f79 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,3 +1,3 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.3")
diff --git a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
index 9ace263..f427b5a 100644
--- a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
+++ b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
@@ -3,7 +3,7 @@ package core
 import org.bitcoinj.core._
 import util._
 
-import scala.collection.convert.WrapAsScala._
+import scala.collection.JavaConverters.asScalaIterator
 
 // In java that should be implements libs.BlockSource
 trait BitcoinDRawFileBlockSource extends BlockSource
diff --git a/src/main/scala/util/LmdbMap.scala b/src/main/scala/util/LmdbMap.scala
index 7c7bd75..a386216 100644
--- a/src/main/scala/util/LmdbMap.scala
+++ b/src/main/scala/util/LmdbMap.scala
@@ -1,6 +1,6 @@
 package util
 
-import scala.collection.convert.WrapAsScala._
+import scala.collection.JavaConverters.asScalaIterator
 import org.fusesource.lmdbjni._
 import org.fusesource.lmdbjni.Constants._
 import scala.collection.mutable.Map

From 2ebf3d426be30763b42673407d1cba3c3f9ee26e Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 23 Mar 2018 12:24:09 +0100
Subject: [PATCH 23/69] fixed tests after upgrade to scala 2.12

---
 src/test/scala/DisjointSetChecks.scala      | 4 ++--
 src/test/scala/RawReaderPropertySuite.scala | 9 +++------
 2 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/src/test/scala/DisjointSetChecks.scala b/src/test/scala/DisjointSetChecks.scala
index 56a1a78..09f00aa 100644
--- a/src/test/scala/DisjointSetChecks.scala
+++ b/src/test/scala/DisjointSetChecks.scala
@@ -13,12 +13,12 @@ object DSSpecification extends Properties("DisjointSets") {
 
   def setOfPairsGen = for {
     s <- arbitrary[Set[Int]]
-    part <- Gen.choose(1,s.size)
+    part <- Gen.choose(0,s.size)
     e <- s
   } yield (part, e)
 
   def setOfPairsToDisjointSet(pairs: Set[(Int, Int)]) : Set[Set[Int]] =
-    pairs.groupBy(_._1).values.toSet map ((p:Set[(Int, Int)]) => (p map (q => q._2)))
+    pairs.groupBy(_._1).values.toSet map ((p:Set[(Int, Int)]) => (p map (_._2)))
 
   property("Sets of nonempty disjoint Sets can be transformed into union/find structures and back") =
     forAll(setOfPairsGen) { (pairs: Set[(Int,Int)]) =>
diff --git a/src/test/scala/RawReaderPropertySuite.scala b/src/test/scala/RawReaderPropertySuite.scala
index 0a72f60..8d1e423 100755
--- a/src/test/scala/RawReaderPropertySuite.scala
+++ b/src/test/scala/RawReaderPropertySuite.scala
@@ -1,10 +1,7 @@
-import org.scalatest.prop._
 import org.scalatest._
-import core._
-import org.scalacheck.Gen._
-import actions._
+import org.scalatest.prop._
 
-class RawReaderPropertySuite extends PropSpec with ShouldMatchers with PropertyChecks {
+class RawReaderPropertySuite extends PropSpec with Matchers with PropertyChecks {
   /*databaseFile = "blockchain/test.db"
    
   property("populater.end should be the minimum of given end and number of blocks available") 
@@ -18,4 +15,4 @@ class RawReaderPropertySuite extends PropSpec with ShouldMatchers with PropertyC
   	 	
   }
   } */
-}
\ No newline at end of file
+}

From 42b1fc712bc72d9082432d9d3c854c5bb482621b Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Tue, 10 Apr 2018 15:59:03 +0200
Subject: [PATCH 24/69] trying to find the unmatched input problem

---
 build.sbt                                     | 2 +-
 src/main/scala/bge/core/FastBlockReader.scala | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index c09d444..c4960d2 100644
--- a/build.sbt
+++ b/build.sbt
@@ -10,7 +10,7 @@ maintainer := "Bitcoinprivacy <info@bitcoinprivacy.net>"
 
 // additional libraries
 libraryDependencies ++= Seq(
-  "org.bitcoinj" % "bitcoinj-core" % "0.15-SNAPSHOT",
+  "org.bitcoinj" % "bitcoinj-core" % "0.14.7",
   "org.xerial.snappy"%"snappy-java"%"1.1.2.4",
   "org.iq80.leveldb"%"leveldb"%"0.7",
   "org.fusesource.leveldbjni"%"leveldbjni-all"%"1.8",
diff --git a/src/main/scala/bge/core/FastBlockReader.scala b/src/main/scala/bge/core/FastBlockReader.scala
index eaf6930..ff565f3 100644
--- a/src/main/scala/bge/core/FastBlockReader.scala
+++ b/src/main/scala/bge/core/FastBlockReader.scala
@@ -96,7 +96,7 @@ trait FastBlockReader extends BlockReader {
     for (input <- outOfOrderInputMap){
     	log.error(input.toString)
     }
-    assert(unmatchedCount == 0, "unmatched Inputs found")
+    assert(unmatchedCount == 0, unmatchedCount + " unmatched Inputs found")
   }
 
   def saveDataToDB: Unit =

From 7f5dc6c22753ed1953a8d0645b9ebf2f5b937189 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 12 Apr 2018 10:45:53 +0200
Subject: [PATCH 25/69] enable docker:publish and postgres-jdbc upgrade

---
 build.sbt | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/build.sbt b/build.sbt
index c4960d2..927fc73 100644
--- a/build.sbt
+++ b/build.sbt
@@ -10,11 +10,11 @@ maintainer := "Bitcoinprivacy <info@bitcoinprivacy.net>"
 
 // additional libraries
 libraryDependencies ++= Seq(
-  "org.bitcoinj" % "bitcoinj-core" % "0.14.7",
+  "org.bitcoinj" % "bitcoinj-core" % "0.15-SNAPSHOT",
   "org.xerial.snappy"%"snappy-java"%"1.1.2.4",
   "org.iq80.leveldb"%"leveldb"%"0.7",
   "org.fusesource.leveldbjni"%"leveldbjni-all"%"1.8",
-  "org.postgresql" % "postgresql" % "9.4-1200-jdbc41",
+  "org.postgresql" % "postgresql" % "42.2.2",
   "com.typesafe.slick" %% "slick" % "2.1.0",
   "com.typesafe" % "config" % "1.2.1",
   "org.scalacheck" %% "scalacheck" % "1.13.5" % "test",
@@ -31,6 +31,11 @@ libraryDependencies ++= Seq(
 )
 
 enablePlugins(JavaAppPackaging)
+enablePlugins(DockerPlugin)
+
+dockerEntrypoint := Seq("bin/bge", "-Dlogback.configurationFile=/conf/logback.xml", "--", "start")
+dockerUsername := Some("jorgemartinezpizarro")
+daemonUser in Docker := "root"
 
 libraryDependencies ~= { _.map(_.exclude("org.slf4j", "slf4j-simple")) }
 

From 0806f0b7c9340c7c0021848abcf8802fc6c20f28 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Wed, 16 May 2018 20:32:22 +0200
Subject: [PATCH 26/69] several fixes

---
 api/ensime.sbt                                       |  1 +
 api/project/plugins.sbt                              |  4 ++--
 build.sbt                                            |  2 +-
 src/main/scala/bge/Explorer.scala                    | 12 ++++++++++--
 src/main/scala/bge/actions/PopulateBlockReader.scala |  2 +-
 src/main/scala/bge/actions/ResumeClosure.scala       | 12 ++++++------
 .../scala/bge/core/BitcoinDRawFileBlockSource.scala  |  9 ++++-----
 src/main/scala/bge/core/BlockSource.scala            |  1 -
 8 files changed, 25 insertions(+), 18 deletions(-)
 create mode 100644 api/ensime.sbt

diff --git a/api/ensime.sbt b/api/ensime.sbt
new file mode 100644
index 0000000..02ef3dc
--- /dev/null
+++ b/api/ensime.sbt
@@ -0,0 +1 @@
+ensimeScalaVersion in ThisBuild := "2.11.8"
diff --git a/api/project/plugins.sbt b/api/project/plugins.sbt
index 25caeb5..f853f72 100644
--- a/api/project/plugins.sbt
+++ b/api/project/plugins.sbt
@@ -1,6 +1,6 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.3")
 
 addSbtPlugin("com.mojolly.scalate" % "xsbt-scalate-generator" % "0.5.0")
 
diff --git a/build.sbt b/build.sbt
index 927fc73..6958156 100644
--- a/build.sbt
+++ b/build.sbt
@@ -33,7 +33,7 @@ libraryDependencies ++= Seq(
 enablePlugins(JavaAppPackaging)
 enablePlugins(DockerPlugin)
 
-dockerEntrypoint := Seq("bin/bge", "-Dlogback.configurationFile=/conf/logback.xml", "--", "start")
+dockerEntrypoint := Seq("bin/bge", "-Dlogback.configurationFile=/conf/logback.xml", "-Dconfig.file=/conf/bge.conf", "--", "start")
 dockerUsername := Some("jorgemartinezpizarro")
 daemonUser in Docker := "root"
 
diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 03f4f24..f64abbb 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -17,7 +17,7 @@ object Explorer extends App with db.BitcoinDB {
       if (!statsDone || rest.headOption == Some("--force"))
         populate
       Seq("touch",lockFile).!
-      iterateResume()
+      iterateResume(true)
 
     case "populate"::rest             =>
 
@@ -42,7 +42,7 @@ object Explorer extends App with db.BitcoinDB {
 
         Available commands:
 
-         start [--force]: populate if necessary or --force, then resume
+         start [--force]: populate if necessary or --force, then resume --newstats
          stop: ask bge gently to stop at the end of the current iteration. Note that thís can take a few days if you are in the populate phase.
          populate: create the database movements with movements and closures.  Deletes any existing data.
          resume [--newstats]: update the database generated by populate with new incoming data. Works only if populate is already done. The newstats flag forces a regeneration of balance, richlist and stats tables.
@@ -98,6 +98,8 @@ object Explorer extends App with db.BitcoinDB {
     initializeStatsTables
 
     insertStatistics
+
+    if (!peerGroup.isRunning) startBitcoinJ
  
     PopulateBlockReader
   
@@ -186,6 +188,12 @@ object Explorer extends App with db.BitcoinDB {
 
   def populateStats = {
     createBalanceTables
+
+
+
+
+
+
     insertRichestAddresses
     insertRichestClosures
     insertStatistics
diff --git a/src/main/scala/bge/actions/PopulateBlockReader.scala b/src/main/scala/bge/actions/PopulateBlockReader.scala
index a3945b3..f70a349 100644
--- a/src/main/scala/bge/actions/PopulateBlockReader.scala
+++ b/src/main/scala/bge/actions/PopulateBlockReader.scala
@@ -5,7 +5,7 @@ import core._
 import util._
 import Hash._
 
-object PopulateBlockReader extends FastBlockReader with BitcoinDRawFileBlockSource {
+object PopulateBlockReader extends FastBlockReader with PeerSource {
   // txhash -> ((index -> (address,value)),blockIn)
   // need to override this here to get the specialized type
   override lazy val table: LmdbMap = LmdbMap.create("utxos")
diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index c4799bc..11dfbf5 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -13,7 +13,7 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
   override lazy val unionFindTable = new ClosureMap(table)
 
   lazy val changedReps = Map[Hash,Set[Hash]]()
-  
+
 
   override def insertInputsIntoTree(addressList: Iterable[Hash], tree: DisjointSets[Hash]): DisjointSets[Hash] =
   {
@@ -46,16 +46,16 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
       }
       else changedReps += (newRep -> (oldReps + oldRep) )
     }
-    
+
 
     DB withSession { implicit session =>
 
-      for ((address, oldRepOpt) <- pairList)
+      for ((address, oldRepOpt) <- pairList.distinct)
         oldRepOpt match
         {
           case None =>
             insertAddress(address,newRep)
-        
+            recordChangedRep(address,newRep)
           case Some(oldRep) if (oldRep != newRep) =>
             // if representant new, update everything that had the old one
             val updateQuery = for(p <- addresses if p.representant === hashToArray(oldRep)) yield p.representant
@@ -68,10 +68,10 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
 
     result
 
-    
+
   }
 
-  
+
   override def saveTree(tree: DisjointSets[Hash]): Int =  // don't replace the postgres DB
   {
     val no = tree.elements.size - startTableSize // return the number of new elements in the union-find structure
diff --git a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
index f427b5a..8386c9b 100644
--- a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
+++ b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
@@ -12,12 +12,11 @@ trait BitcoinDRawFileBlockSource extends BlockSource
 
     startBitcoinJ
     log.info("starting")
-    val almostCurrentChain = if (maxPopulate == 0)
-	getCurrentLongestChainFromBlockCount
-    else if (maxPopulate > 0)
+    val almostCurrentChain =
+      if (maxPopulate == 0) getCurrentLongestChainFromBlockCount
+      else if (maxPopulate > 0)
         getCurrentLongestChainFromBlockCount.take(maxPopulate)
-    else
-	throw new Exception(s"invalid parameter bitcoin.maxPopulate = '$maxPopulate'")
+      else throw new Exception(s"invalid parameter bitcoin.maxPopulate = '$maxPopulate'")
 
     val blockMap = almostCurrentChain toMap
 
diff --git a/src/main/scala/bge/core/BlockSource.scala b/src/main/scala/bge/core/BlockSource.scala
index a42c738..6c323b1 100644
--- a/src/main/scala/bge/core/BlockSource.scala
+++ b/src/main/scala/bge/core/BlockSource.scala
@@ -23,7 +23,6 @@ trait BlockSource extends db.BitcoinDB {
         previousElement
     }._1
 
-
     val hashes = blockList map (_._1)
     assert(hashes.length == hashes.distinct.length, "duplicate block hashes in blockChain (bitcoinJ/levelDB problems?)")
 

From ac09d450ae2f5afedee9c03d2295785873db6890 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 17 May 2018 10:15:52 +0200
Subject: [PATCH 27/69] another fix in updateBalances.

---
 src/main/scala/db/BitcoinDB.scala | 71 ++++++++++++++++---------------
 1 file changed, 37 insertions(+), 34 deletions(-)

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 267afc5..adcb3db 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -156,52 +156,55 @@ trait BitcoinDB {
       }
       // Hash.zero(0) is not an address
       session.withTransaction {
-      val adsAndBalances = for ((address, change) <- changedAddresses - Hash.zero(0))
-        yield (address,
-        balances.filter(_.address === Hash.hashToArray(address)).
-        map(_.balance).firstOption.getOrElse(0L) + change)
+        val adsAndBalances = for ((address, change) <- changedAddresses - Hash.zero(0))
+                             yield (address,
+                                    balances.filter(_.address === Hash.hashToArray(address)).
+                                      map(_.balance).firstOption.getOrElse(0L) + change)
 
-      updateAdsBalancesTable(adsAndBalances, balances)
+        updateAdsBalancesTable(adsAndBalances, balances)
 
-      val table = LmdbMap.open("closures")
-      val unionFindTable = new ClosureMap(table)
-      val closures = new DisjointSets[Hash](unionFindTable)
+        // insert new trivial closures into closure_balance before accounting for changedReps!
+        val newRepsAndBalances = (changedAddresses - Hash.zero(0)) filter {case (address, _) => (addresses.filter(_.hash === Hash.hashToArray(address)).firstOption == None)}
+        updateAdsBalancesTable(newRepsAndBalances, closureBalances)                         
 
-      val repsAndChanges: collection.mutable.Map[Hash, Long] = collection.mutable.Map()
+        val table = LmdbMap.open("closures")
+        val unionFindTable = new ClosureMap(table)
+        val closures = new DisjointSets[Hash](unionFindTable)
 
-      for ((address, change) <- changedAddresses - Hash.zero(0) ) {
-               val rep = closures.find(address)._1.getOrElse(address)
-               val newBalance = repsAndChanges.getOrElse(rep, 0L) + change
-               repsAndChanges += (rep -> newBalance)
-      } 
+        val repsAndChanges: collection.mutable.Map[Hash, Long] = collection.mutable.Map()
 
-      // don't forget the new reps that had no balance change!
-      for ((rep,_) <- changedReps)
-        repsAndChanges += (rep -> repsAndChanges.getOrElse(rep, 0L))
+        for ((address, change) <- changedAddresses - Hash.zero(0) ) {
+          val rep = closures.find(address)._1.getOrElse(address)
+          val newBalance = repsAndChanges.getOrElse(rep, 0L) + change
+          repsAndChanges += (rep -> newBalance)
+        }
+
+        // don't forget the new reps that had no balance change!
+        for ((rep,_) <- changedReps)
+          repsAndChanges += (rep -> repsAndChanges.getOrElse(rep, 0L))
   
-      val repsAndBalances: Map[Hash, Long] =
-        for {
-          (rep, change) <- repsAndChanges 
-          oldReps = (changedReps.getOrElse(rep, Set()) + rep).map(Hash.hashToArray(_))
-        } yield (rep,
-          closureBalances.filter(_.address inSetBind oldReps).
-          map(_.balance).sum.run.getOrElse(0L) + change)
+        val repsAndBalances: Map[Hash, Long] =
+          for {
+            (rep, change) <- repsAndChanges
+            oldReps = (changedReps.getOrElse(rep, Set()) + rep).map(Hash.hashToArray(_))
+          } yield (rep,
+                   closureBalances.filter(_.address inSetBind oldReps).
+                     map(_.balance).sum.run.getOrElse(0L) + change)
 
-      updateAdsBalancesTable(repsAndBalances, closureBalances)
+        updateAdsBalancesTable(repsAndBalances, closureBalances)
 
-      // delete all oldReps that have been unified into new ones
-      val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
-      closureBalances.filter(_.address inSet toDelete).delete
+        // delete all oldReps that have been unified into new ones
+        val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
+        closureBalances.filter(_.address inSet toDelete).delete
 
-      table.close
+        table.close
 
-      log.info("%s balances updated in %s s, %s µs per address "
-        format
-        (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
+        log.info("%s balances updated in %s s, %s µs per address "
+                   format
+                   (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
 
-        
       }
-     }
+    }
   }
 
   def insertRichestClosures = {

From d3d62c56a8f84bdf89817a1eec92b1e038905701 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 17 May 2018 16:14:36 +0200
Subject: [PATCH 28/69] fix updateBalances

---
 src/main/scala/bge/actions/ResumeClosure.scala | 2 +-
 src/main/scala/db/BitcoinDB.scala              | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index 11dfbf5..884e23f 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -55,7 +55,7 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
         {
           case None =>
             insertAddress(address,newRep)
-            recordChangedRep(address,newRep)
+            recordChangedRep(newRep,address)
           case Some(oldRep) if (oldRep != newRep) =>
             // if representant new, update everything that had the old one
             val updateQuery = for(p <- addresses if p.representant === hashToArray(oldRep)) yield p.representant
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index adcb3db..be62761 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -163,10 +163,6 @@ trait BitcoinDB {
 
         updateAdsBalancesTable(adsAndBalances, balances)
 
-        // insert new trivial closures into closure_balance before accounting for changedReps!
-        val newRepsAndBalances = (changedAddresses - Hash.zero(0)) filter {case (address, _) => (addresses.filter(_.hash === Hash.hashToArray(address)).firstOption == None)}
-        updateAdsBalancesTable(newRepsAndBalances, closureBalances)                         
-
         val table = LmdbMap.open("closures")
         val unionFindTable = new ClosureMap(table)
         val closures = new DisjointSets[Hash](unionFindTable)

From d0f32657fe61fd0e3905e33d1a3cb09a71e8b9fb Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sat, 19 May 2018 15:42:53 +0200
Subject: [PATCH 29/69] Added bge integration tests (required of docker
 environment, this need to be discussed)

---
 src/test/resources/application.conf         | 21 +++++++++++++
 src/test/scala/RawReaderPropertySuite.scala | 34 ++++++++++++---------
 test.sh                                     |  5 +++
 3 files changed, 46 insertions(+), 14 deletions(-)
 create mode 100644 src/test/resources/application.conf
 create mode 100755 test.sh

diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf
new file mode 100644
index 0000000..990af7c
--- /dev/null
+++ b/src/test/resources/application.conf
@@ -0,0 +1,21 @@
+populateTransactionSize=100000 // movemements
+closureReadSize=1000 // blocks
+closureTransactionSize=50000
+balanceTransactionSize=50000
+richlistSize=1000
+dataDir = "/root/Bitcoin-Graph-Explorer/src/test/blockchain/"
+levelDBFile = ${dataDir}"level.db"
+lockFile = ${dataDir}"lock"
+dustLimit=100000
+databaseName="postgres2"
+username = "root"
+password = "trivial"
+host="localhost:5433"
+jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
+mdb.path = ${dataDir}"mdb/"
+mdb.transactionSize = 100000
+network=regtest # accepts main, testnet, regtest
+resumeBlockSize=10000000 # depending on the available ram
+bitcoin.ip=localhost
+bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
+balanceUpdateLimit=70000 
diff --git a/src/test/scala/RawReaderPropertySuite.scala b/src/test/scala/RawReaderPropertySuite.scala
index 8d1e423..cedf04f 100755
--- a/src/test/scala/RawReaderPropertySuite.scala
+++ b/src/test/scala/RawReaderPropertySuite.scala
@@ -1,18 +1,24 @@
+import sys.process._
+import util._
 import org.scalatest._
-import org.scalatest.prop._
 
-class RawReaderPropertySuite extends PropSpec with Matchers with PropertyChecks {
-  /*databaseFile = "blockchain/test.db"
-   
-  property("populater.end should be the minimum of given end and number of blocks available") 
-  {
-   forAll (choose(0,280000), minSuccessful(1)) { (n:Int) =>
-  	 {
-      val populater = new BlocksReader(List(n.toString,"init"))
-      populater.start should be(0) 
-      populater.end should be (n) // TODO: This is not true. We need the number of blocks available and this is also at least a problem when 0 is given
-  	 }
-  	 	
+class PopulateSpec extends FlatSpec with Matchers {
+  val BITCOIN = "docker exec --user bitcoin regtestbge_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
+  def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
+  def resetRegtest = "/root/Bitcoin-Graph-Explorer/test.sh" ! ProcessLogger(_ => ())
+
+  "Populate" should "safe 5 blocks as the blockchain contains 5 blocks" in {
+    resetRegtest
+    gen(5)
+    Explorer.populate
+    Explorer.blockCount should be (6)
+  }
+
+  "Resume" should "add another 10 blocks after being added to blockchain" in {
+    gen(10)
+    Explorer.resume
+    Explorer.blockCount should be (16)
+    
   }
-  } */
 }
+
diff --git a/test.sh b/test.sh
new file mode 100755
index 0000000..48ce933
--- /dev/null
+++ b/test.sh
@@ -0,0 +1,5 @@
+rm /root/files-regtest/bitcoin/regtest/ -rf
+rm /root/Bitcoin-Graph-Explorer/src/test/blockchain/ -rf
+cd /root/regtest-bge
+make stop start
+sleep 5s

From 78734867b946605853690f199365163dde194a26 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sat, 19 May 2018 15:52:49 +0200
Subject: [PATCH 30/69] move test script to test folder. rename old spec to bge
 spec

---
 test.sh => src/test/resources/test.sh                      | 0
 .../scala/{RawReaderPropertySuite.scala => BGESpec.scala}  | 7 +++++--
 2 files changed, 5 insertions(+), 2 deletions(-)
 rename test.sh => src/test/resources/test.sh (100%)
 rename src/test/scala/{RawReaderPropertySuite.scala => BGESpec.scala} (64%)

diff --git a/test.sh b/src/test/resources/test.sh
similarity index 100%
rename from test.sh
rename to src/test/resources/test.sh
diff --git a/src/test/scala/RawReaderPropertySuite.scala b/src/test/scala/BGESpec.scala
similarity index 64%
rename from src/test/scala/RawReaderPropertySuite.scala
rename to src/test/scala/BGESpec.scala
index cedf04f..ea854e9 100755
--- a/src/test/scala/RawReaderPropertySuite.scala
+++ b/src/test/scala/BGESpec.scala
@@ -2,10 +2,13 @@ import sys.process._
 import util._
 import org.scalatest._
 
-class PopulateSpec extends FlatSpec with Matchers {
+class BGESpec extends FlatSpec with Matchers {
   val BITCOIN = "docker exec --user bitcoin regtestbge_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
+  // fixme: to use an script is dirty but we need to access to bitcoincli which lives in a container
   def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
-  def resetRegtest = "/root/Bitcoin-Graph-Explorer/test.sh" ! ProcessLogger(_ => ())
+  def resetRegtest = "/root/Bitcoin-Graph-Explorer/src/test/resources/test.sh" ! ProcessLogger(_ => ()) 
+  // fixme: to use an script is dirty but i need to restart some containers...
+
 
   "Populate" should "safe 5 blocks as the blockchain contains 5 blocks" in {
     resetRegtest

From 30a4f3c8037fd412a3349aeb45461c6fd9746e2e Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 08:33:52 +0200
Subject: [PATCH 31/69] add docker environment for testing

---
 .gitignore                             |   4 +
 src/test/docker/Makefile               |  32 ++
 src/test/docker/bitcoin/bitcoin.conf   |   3 +
 src/test/docker/docker-compose.yml     |  45 ++
 src/test/docker/postgres/postgres.conf | 541 +++++++++++++++++++++++++
 src/test/docker/reset.sh               |  16 +
 src/test/resources/application.conf    |   2 +-
 src/test/resources/test.sh             |   5 -
 src/test/scala/BGESpec.scala           |   5 +-
 9 files changed, 644 insertions(+), 9 deletions(-)
 create mode 100644 src/test/docker/Makefile
 create mode 100644 src/test/docker/bitcoin/bitcoin.conf
 create mode 100644 src/test/docker/docker-compose.yml
 create mode 100644 src/test/docker/postgres/postgres.conf
 create mode 100755 src/test/docker/reset.sh
 delete mode 100755 src/test/resources/test.sh

diff --git a/.gitignore b/.gitignore
index 37e83bc..6252cb2 100755
--- a/.gitignore
+++ b/.gitignore
@@ -10,3 +10,7 @@ neodb/
 src/main/resources/application.conf
 .history
 blockchain
+src/test/data/blockchain/*
+src/test/data/psql/*
+src/test/data/bitcoin/*
+
diff --git a/src/test/docker/Makefile b/src/test/docker/Makefile
new file mode 100644
index 0000000..f268426
--- /dev/null
+++ b/src/test/docker/Makefile
@@ -0,0 +1,32 @@
+export VERSION=SNAPSHOT
+export REPOSITORY=jorgemartinezpizarro
+export FOLDER=/root/Bitcoin-Graph-Explorer/src/test/docker
+## start
+start:
+	docker-compose up -d
+
+## stop
+stop:
+	docker-compose kill
+
+## clean
+clean:
+	docker-compose rm
+
+##logs
+logs:
+	docker-compose logs --tail="all" --follow bitcoin postgres
+
+## show this help screen
+help:
+	@printf "Available targets\n\n"
+	@awk '/^[a-zA-Z\-\_0-9]+:/ { \
+		helpMessage = match(lastLine, /^## (.*)/); \
+		if (helpMessage) { \
+			helpCommand = substr($$1, 0, index($$1, ":")); \
+			helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \
+			printf "%-15s %s\n", helpCommand, helpMessage; \
+		} \
+	} \
+	{ lastLine = $$0 }' $(MAKEFILE_LIST)
+
diff --git a/src/test/docker/bitcoin/bitcoin.conf b/src/test/docker/bitcoin/bitcoin.conf
new file mode 100644
index 0000000..e5010ba
--- /dev/null
+++ b/src/test/docker/bitcoin/bitcoin.conf
@@ -0,0 +1,3 @@
+rpcuser='username'
+rpcpassword='password'
+regtest=1
diff --git a/src/test/docker/docker-compose.yml b/src/test/docker/docker-compose.yml
new file mode 100644
index 0000000..0341fa9
--- /dev/null
+++ b/src/test/docker/docker-compose.yml
@@ -0,0 +1,45 @@
+version: "3"
+services: 
+  bitcoin:
+    image: ruimarinho/bitcoin-core:0.16.0
+    networks: 
+      bge_network:
+        ipv4_address: 172.25.1.2 
+    ports:
+      - "18444:18444"
+    command: 
+      -printtoconsole
+      -regtest=1
+      -rest
+      -rpcallowip=173.249.18.12
+      -rpcpassword=bar
+      -rpcport=18333
+      -rpcuser=foo
+      -server
+    volumes:
+      - "/root/Bitcoin-Graph-Explorer/src/test/data/bitcoin:/home/bitcoin/.bitcoin"
+      - "$FOLDER/bitcoin/bitcoin.conf:/etc/bitcoin/bitcoin.conf"
+  postgres:
+    image: postgres:10.1
+    networks:
+      bge_network:
+        ipv4_address: 172.25.1.3
+    expose:
+      - 5433
+    ports:
+      - "5433:5433"
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: trivial
+      POSTGRES_DB: postgres2
+    volumes:
+      - "/root/Bitcoin-Graph-Explorer/src/test/data/psql:/var/lib/postgresql/data"
+      - "$FOLDER/postgres/postgres.conf:/var/lib/postgresql/data/postgresql.conf"
+networks:
+  bge_network:
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+      -
+        subnet: 172.25.1.0/24
diff --git a/src/test/docker/postgres/postgres.conf b/src/test/docker/postgres/postgres.conf
new file mode 100644
index 0000000..54ef29f
--- /dev/null
+++ b/src/test/docker/postgres/postgres.conf
@@ -0,0 +1,541 @@
+# -----------------------------
+# PostgreSQL configuration file
+# -----------------------------
+#
+# This file consists of lines of the form:
+#
+#   name = value
+#
+# (The "=" is optional.)  Whitespace may be used.  Comments are introduced with
+# "#" anywhere on a line.  The complete list of parameter names and allowed
+# values can be found in the PostgreSQL documentation.
+#
+# The commented-out settings shown in this file represent the default values.
+# Re-commenting a setting is NOT sufficient to revert it to the default value;
+# you need to reload the server.
+#
+# This file is read on server startup and when the server receives a SIGHUP
+# signal.  If you edit the file on a running system, you have to SIGHUP the
+# server for the changes to take effect, or use "pg_ctl reload".  Some
+# parameters, which are marked below, require a server shutdown and restart to
+# take effect.
+#
+# Any parameter can also be given as a command-line option to the server, e.g.,
+# "postgres -c log_connections=on".  Some parameters can be changed at run time
+# with the "SET" SQL command.
+#
+# Memory units:  kB = kilobytes        Time units:  ms  = milliseconds
+#                MB = megabytes                     s   = seconds
+#                GB = gigabytes                     min = minutes
+#                                                   h   = hours
+#                                                   d   = days
+
+
+#------------------------------------------------------------------------------
+# FILE LOCATIONS
+#------------------------------------------------------------------------------
+
+# The default values of these variables are driven from the -D command-line
+# option or PGDATA environment variable, represented here as ConfigDir.
+
+#data_directory = 'ConfigDir'		# use data in another directory
+					# (change requires restart)
+#hba_file = 'ConfigDir/pg_hba.conf'	# host-based authentication file
+					# (change requires restart)
+#ident_file = 'ConfigDir/pg_ident.conf'	# ident configuration file
+					# (change requires restart)
+
+# If external_pid_file is not explicitly set, no extra PID file is written.
+#external_pid_file = '(none)'		# write an extra PID file
+					# (change requires restart)
+
+
+#------------------------------------------------------------------------------
+# CONNECTIONS AND AUTHENTICATION
+#------------------------------------------------------------------------------
+
+# - Connection Settings -
+
+#listen_addresses = 'localhost'		# what IP address(es) to listen on;
+					# comma-separated list of addresses;
+					# defaults to 'localhost', '*' = all
+					# (change requires restart)
+#port = 5432				# (change requires restart)
+#max_connections = 100			# (change requires restart)
+# Note:  Increasing max_connections costs ~400 bytes of shared memory per 
+# connection slot, plus lock space (see max_locks_per_transaction).
+#superuser_reserved_connections = 3	# (change requires restart)
+#unix_socket_directory = ''		# (change requires restart)
+#unix_socket_group = ''			# (change requires restart)
+#unix_socket_permissions = 0777		# begin with 0 to use octal notation
+					# (change requires restart)
+#bonjour = off				# advertise server via Bonjour
+					# (change requires restart)
+#bonjour_name = ''			# defaults to the computer name
+					# (change requires restart)
+
+# - Security and Authentication -
+
+#authentication_timeout = 1min		# 1s-600s
+#ssl = off				# (change requires restart)
+#ssl_ciphers = 'ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH'	# allowed SSL ciphers
+					# (change requires restart)
+#ssl_renegotiation_limit = 512MB	# amount of data between renegotiations
+#password_encryption = on
+#db_user_namespace = off
+
+# Kerberos and GSSAPI
+#krb_server_keyfile = ''
+#krb_srvname = 'postgres'		# (Kerberos only)
+#krb_caseins_users = off
+
+# - TCP Keepalives -
+# see "man 7 tcp" for details
+
+#tcp_keepalives_idle = 0		# TCP_KEEPIDLE, in seconds;
+					# 0 selects the system default
+#tcp_keepalives_interval = 0		# TCP_KEEPINTVL, in seconds;
+					# 0 selects the system default
+#tcp_keepalives_count = 0		# TCP_KEEPCNT;
+					# 0 selects the system default
+
+
+#------------------------------------------------------------------------------
+# RESOURCE USAGE (except WAL)
+#------------------------------------------------------------------------------
+
+# - Memory -
+
+#shared_buffers = 32MB			# min 128kB
+					# (change requires restart)
+#temp_buffers = 8MB			# min 800kB
+#max_prepared_transactions = 0		# zero disables the feature
+					# (change requires restart)
+# Note:  Increasing max_prepared_transactions costs ~600 bytes of shared memory
+# per transaction slot, plus lock space (see max_locks_per_transaction).
+# It is not advisable to set max_prepared_transactions nonzero unless you
+# actively intend to use prepared transactions.
+#work_mem = 1MB				# min 64kB
+#maintenance_work_mem = 16MB		# min 1MB
+#max_stack_depth = 2MB			# min 100kB
+
+# - Kernel Resource Usage -
+
+#max_files_per_process = 1000		# min 25
+					# (change requires restart)
+#shared_preload_libraries = ''		# (change requires restart)
+
+# - Cost-Based Vacuum Delay -
+
+#vacuum_cost_delay = 0ms		# 0-100 milliseconds
+#vacuum_cost_page_hit = 1		# 0-10000 credits
+#vacuum_cost_page_miss = 10		# 0-10000 credits
+#vacuum_cost_page_dirty = 20		# 0-10000 credits
+#vacuum_cost_limit = 200		# 1-10000 credits
+
+# - Background Writer -
+
+#bgwriter_delay = 200ms			# 10-10000ms between rounds
+#bgwriter_lru_maxpages = 100		# 0-1000 max buffers written/round
+#bgwriter_lru_multiplier = 2.0		# 0-10.0 multipler on buffers scanned/round
+
+# - Asynchronous Behavior -
+
+#effective_io_concurrency = 1		# 1-1000. 0 disables prefetching
+
+
+#------------------------------------------------------------------------------
+# WRITE AHEAD LOG
+#------------------------------------------------------------------------------
+
+# - Settings -
+
+#wal_level = minimal			# minimal, archive, or hot_standby
+					# (change requires restart)
+#fsync = on				# turns forced synchronization on or off
+#synchronous_commit = on		# immediate fsync at commit
+#wal_sync_method = fsync		# the default is the first option 
+					# supported by the operating system:
+					#   open_datasync
+					#   fdatasync (default on Linux)
+					#   fsync
+					#   fsync_writethrough
+					#   open_sync
+#full_page_writes = on			# recover from partial page writes
+#wal_buffers = 64kB			# min 32kB
+					# (change requires restart)
+#wal_writer_delay = 200ms		# 1-10000 milliseconds
+
+#commit_delay = 0			# range 0-100000, in microseconds
+#commit_siblings = 5			# range 1-1000
+
+# - Checkpoints -
+
+#checkpoint_segments = 3		# in logfile segments, min 1, 16MB each
+#checkpoint_timeout = 5min		# range 30s-1h
+#checkpoint_completion_target = 0.5	# checkpoint target duration, 0.0 - 1.0
+#checkpoint_warning = 30s		# 0 disables
+
+# - Archiving -
+
+#archive_mode = off		# allows archiving to be done
+				# (change requires restart)
+#archive_command = ''		# command to use to archive a logfile segment
+#archive_timeout = 0		# force a logfile segment switch after this
+				# number of seconds; 0 disables
+
+# - Streaming Replication -
+
+#max_wal_senders = 0		# max number of walsender processes
+				# (change requires restart)
+#wal_sender_delay = 200ms	# walsender cycle time, 1-10000 milliseconds
+#wal_keep_segments = 0		# in logfile segments, 16MB each; 0 disables
+#vacuum_defer_cleanup_age = 0	# number of xacts by which cleanup is delayed
+
+# - Standby Servers -
+
+#hot_standby = off			# "on" allows queries during recovery
+					# (change requires restart)
+#max_standby_archive_delay = 30s	# max delay before canceling queries
+					# when reading WAL from archive;
+					# -1 allows indefinite delay
+#max_standby_streaming_delay = 30s	# max delay before canceling queries
+					# when reading streaming WAL;
+					# -1 allows indefinite delay
+
+
+#------------------------------------------------------------------------------
+# QUERY TUNING
+#------------------------------------------------------------------------------
+
+# - Planner Method Configuration -
+
+#enable_bitmapscan = on
+#enable_hashagg = on
+#enable_hashjoin = on
+#enable_indexscan = on
+#enable_material = on
+#enable_mergejoin = on
+#enable_nestloop = on
+#enable_seqscan = on
+#enable_sort = on
+#enable_tidscan = on
+
+# - Planner Cost Constants -
+
+#seq_page_cost = 1.0			# measured on an arbitrary scale
+#random_page_cost = 4.0			# same scale as above
+#cpu_tuple_cost = 0.01			# same scale as above
+#cpu_index_tuple_cost = 0.005		# same scale as above
+#cpu_operator_cost = 0.0025		# same scale as above
+#effective_cache_size = 128MB
+
+# - Genetic Query Optimizer -
+
+#geqo = on
+#geqo_threshold = 12
+#geqo_effort = 5			# range 1-10
+#geqo_pool_size = 0			# selects default based on effort
+#geqo_generations = 0			# selects default based on effort
+#geqo_selection_bias = 2.0		# range 1.5-2.0
+#geqo_seed = 0.0			# range 0.0-1.0
+
+# - Other Planner Options -
+
+#default_statistics_target = 100	# range 1-10000
+#constraint_exclusion = partition	# on, off, or partition
+#cursor_tuple_fraction = 0.1		# range 0.0-1.0
+#from_collapse_limit = 8
+#join_collapse_limit = 8		# 1 disables collapsing of explicit 
+					# JOIN clauses
+
+
+#------------------------------------------------------------------------------
+# ERROR REPORTING AND LOGGING
+#------------------------------------------------------------------------------
+
+# - Where to Log -
+
+#log_destination = 'stderr'		# Valid values are combinations of
+					# stderr, csvlog, syslog, and eventlog,
+					# depending on platform.  csvlog
+					# requires logging_collector to be on.
+
+# This is used when logging to stderr:
+#logging_collector = off		# Enable capturing of stderr and csvlog
+					# into log files. Required to be on for
+					# csvlogs.
+					# (change requires restart)
+
+# These are only used if logging_collector is on:
+#log_directory = 'pg_log'		# directory where log files are written,
+					# can be absolute or relative to PGDATA
+#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'	# log file name pattern,
+					# can include strftime() escapes
+#log_truncate_on_rotation = off		# If on, an existing log file of the
+					# same name as the new log file will be
+					# truncated rather than appended to.
+					# But such truncation only occurs on
+					# time-driven rotation, not on restarts
+					# or size-driven rotation.  Default is
+					# off, meaning append to existing files
+					# in all cases.
+#log_rotation_age = 1d			# Automatic rotation of logfiles will
+					# happen after that time.  0 disables.
+#log_rotation_size = 10MB		# Automatic rotation of logfiles will 
+					# happen after that much log output.
+					# 0 disables.
+
+# These are relevant when logging to syslog:
+#syslog_facility = 'LOCAL0'
+#syslog_ident = 'postgres'
+
+#silent_mode = off			# Run server silently.
+					# DO NOT USE without syslog or
+					# logging_collector
+					# (change requires restart)
+
+
+# - When to Log -
+
+#client_min_messages = notice		# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   log
+					#   notice
+					#   warning
+					#   error
+
+#log_min_messages = warning		# values in order of decreasing detail:
+					#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+					#   info
+					#   notice
+					#   warning
+					#   error
+					#   log
+					#   fatal
+					#   panic
+
+#log_min_error_statement = error	# values in order of decreasing detail:
+				 	#   debug5
+					#   debug4
+					#   debug3
+					#   debug2
+					#   debug1
+				 	#   info
+					#   notice
+					#   warning
+					#   error
+					#   log
+					#   fatal
+					#   panic (effectively off)
+
+#log_min_duration_statement = -1	# -1 is disabled, 0 logs all statements
+					# and their durations, > 0 logs only
+					# statements running at least this number
+					# of milliseconds
+
+
+# - What to Log -
+
+#debug_print_parse = off
+#debug_print_rewritten = off
+#debug_print_plan = off
+#debug_pretty_print = on
+#log_checkpoints = off
+#log_connections = off
+#log_disconnections = off
+#log_duration = off
+#log_error_verbosity = default		# terse, default, or verbose messages
+#log_hostname = off
+#log_line_prefix = ''			# special values:
+					#   %a = application name
+					#   %u = user name
+					#   %d = database name
+					#   %r = remote host and port
+					#   %h = remote host
+					#   %p = process ID
+					#   %t = timestamp without milliseconds
+					#   %m = timestamp with milliseconds
+					#   %i = command tag
+					#   %e = SQL state
+					#   %c = session ID
+					#   %l = session line number
+					#   %s = session start timestamp
+					#   %v = virtual transaction ID
+					#   %x = transaction ID (0 if none)
+					#   %q = stop here in non-session
+					#        processes
+					#   %% = '%'
+					# e.g. '<%u%%%d> '
+#log_lock_waits = off			# log lock waits >= deadlock_timeout
+#log_statement = 'none'			# none, ddl, mod, all
+#log_temp_files = -1			# log temporary files equal or larger
+					# than the specified size in kilobytes;
+					# -1 disables, 0 logs all temp files
+#log_timezone = unknown			# actually, defaults to TZ environment
+					# setting
+
+
+#------------------------------------------------------------------------------
+# RUNTIME STATISTICS
+#------------------------------------------------------------------------------
+
+# - Query/Index Statistics Collector -
+
+#track_activities = on
+#track_counts = on
+#track_functions = none			# none, pl, all
+#track_activity_query_size = 1024 	# (change requires restart)
+#update_process_title = on
+#stats_temp_directory = 'pg_stat_tmp'
+
+
+# - Statistics Monitoring -
+
+#log_parser_stats = off
+#log_planner_stats = off
+#log_executor_stats = off
+#log_statement_stats = off
+
+
+#------------------------------------------------------------------------------
+# AUTOVACUUM PARAMETERS
+#------------------------------------------------------------------------------
+
+#autovacuum = on			# Enable autovacuum subprocess?  'on' 
+					# requires track_counts to also be on.
+#log_autovacuum_min_duration = -1	# -1 disables, 0 logs all actions and
+					# their durations, > 0 logs only
+					# actions running at least this number
+					# of milliseconds.
+#autovacuum_max_workers = 3		# max number of autovacuum subprocesses
+					# (change requires restart)
+#autovacuum_naptime = 1min		# time between autovacuum runs
+#autovacuum_vacuum_threshold = 50	# min number of row updates before
+					# vacuum
+#autovacuum_analyze_threshold = 50	# min number of row updates before 
+					# analyze
+#autovacuum_vacuum_scale_factor = 0.2	# fraction of table size before vacuum
+#autovacuum_analyze_scale_factor = 0.1	# fraction of table size before analyze
+#autovacuum_freeze_max_age = 200000000	# maximum XID age before forced vacuum
+					# (change requires restart)
+#autovacuum_vacuum_cost_delay = 20ms	# default vacuum cost delay for
+					# autovacuum, in milliseconds;
+					# -1 means use vacuum_cost_delay
+#autovacuum_vacuum_cost_limit = -1	# default vacuum cost limit for
+					# autovacuum, -1 means use
+					# vacuum_cost_limit
+
+
+#------------------------------------------------------------------------------
+# CLIENT CONNECTION DEFAULTS
+#------------------------------------------------------------------------------
+
+# - Statement Behavior -
+
+#search_path = '"$user",public'		# schema names
+#default_tablespace = ''		# a tablespace name, '' uses the default
+#temp_tablespaces = ''			# a list of tablespace names, '' uses
+					# only default tablespace
+#check_function_bodies = on
+#default_transaction_isolation = 'read committed'
+#default_transaction_read_only = off
+#session_replication_role = 'origin'
+#statement_timeout = 0			# in milliseconds, 0 is disabled
+#vacuum_freeze_min_age = 50000000
+#vacuum_freeze_table_age = 150000000
+#bytea_output = 'hex'			# hex, escape
+#xmlbinary = 'base64'
+#xmloption = 'content'
+
+# - Locale and Formatting -
+
+#datestyle = 'iso, mdy'
+#intervalstyle = 'postgres'
+#timezone = unknown			# actually, defaults to TZ environment
+					# setting
+#timezone_abbreviations = 'Default'     # Select the set of available time zone
+					# abbreviations.  Currently, there are
+					#   Default
+					#   Australia
+					#   India
+					# You can create your own file in
+					# share/timezonesets/.
+#extra_float_digits = 0			# min -15, max 3
+#client_encoding = sql_ascii		# actually, defaults to database
+					# encoding
+
+# These settings are initialized by initdb, but they can be changed.
+#lc_messages = 'C'			# locale for system error message
+					# strings
+#lc_monetary = 'C'			# locale for monetary formatting
+#lc_numeric = 'C'			# locale for number formatting
+#lc_time = 'C'				# locale for time formatting
+
+# default configuration for text search
+#default_text_search_config = 'pg_catalog.simple'
+
+# - Other Defaults -
+
+#dynamic_library_path = '$libdir'
+#local_preload_libraries = ''
+
+
+#------------------------------------------------------------------------------
+# LOCK MANAGEMENT
+#------------------------------------------------------------------------------
+
+#deadlock_timeout = 1s
+#max_locks_per_transaction = 64		# min 10
+					# (change requires restart)
+# Note:  Each lock table slot uses ~270 bytes of shared memory, and there are
+# max_locks_per_transaction * (max_connections + max_prepared_transactions)
+# lock table slots.
+
+
+#------------------------------------------------------------------------------
+# VERSION/PLATFORM COMPATIBILITY
+#------------------------------------------------------------------------------
+
+# - Previous PostgreSQL Versions -
+
+#array_nulls = on
+#backslash_quote = safe_encoding	# on, off, or safe_encoding
+#default_with_oids = off
+#escape_string_warning = on
+#lo_compat_privileges = off
+#sql_inheritance = on
+#standard_conforming_strings = off
+#synchronize_seqscans = on
+
+# - Other Platforms and Clients -
+
+#transform_null_equals = off
+
+
+#------------------------------------------------------------------------------
+# CUSTOMIZED OPTIONS
+#------------------------------------------------------------------------------
+
+#custom_variable_classes = ''		# list of custom variable class names
+
+shared_buffers = 8GB
+effective_cache_size = 24GB
+work_mem = 128MB
+maintenance_work_mem = 8GB
+random_page_cost = 1.5
+effective_io_concurrency = 5
+checkpoint_completion_target=0.9
+min_wal_size = 1GB
+max_wal_size = 2GB
+
+listen_addresses = '*'
+port = 5433
diff --git a/src/test/docker/reset.sh b/src/test/docker/reset.sh
new file mode 100755
index 0000000..f15dfab
--- /dev/null
+++ b/src/test/docker/reset.sh
@@ -0,0 +1,16 @@
+rm /root/Bitcoin-Graph-Explorer/src/test/data/bitcoin/regtest/ -rf
+rm /root/Bitcoin-Graph-Explorer/src/test/data/blockchain/ -rf
+cd /root/Bitcoin-Graph-Explorer/src/test/docker
+make stop start
+echo "waiting for components to be ready"
+echo "5"
+sleep 1s
+echo "4"
+sleep 1s
+echo "3"
+sleep 1s
+echo "2"
+sleep 1s
+echo "1"
+sleep 1s
+echo "done sleeping"
diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf
index 990af7c..0f1cfde 100644
--- a/src/test/resources/application.conf
+++ b/src/test/resources/application.conf
@@ -3,7 +3,7 @@ closureReadSize=1000 // blocks
 closureTransactionSize=50000
 balanceTransactionSize=50000
 richlistSize=1000
-dataDir = "/root/Bitcoin-Graph-Explorer/src/test/blockchain/"
+dataDir = "/root/Bitcoin-Graph-Explorer/src/test/data/blockchain/"
 levelDBFile = ${dataDir}"level.db"
 lockFile = ${dataDir}"lock"
 dustLimit=100000
diff --git a/src/test/resources/test.sh b/src/test/resources/test.sh
deleted file mode 100755
index 48ce933..0000000
--- a/src/test/resources/test.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-rm /root/files-regtest/bitcoin/regtest/ -rf
-rm /root/Bitcoin-Graph-Explorer/src/test/blockchain/ -rf
-cd /root/regtest-bge
-make stop start
-sleep 5s
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index ea854e9..8ad89b3 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -3,13 +3,12 @@ import util._
 import org.scalatest._
 
 class BGESpec extends FlatSpec with Matchers {
-  val BITCOIN = "docker exec --user bitcoin regtestbge_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
+  val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
   // fixme: to use an script is dirty but we need to access to bitcoincli which lives in a container
   def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
-  def resetRegtest = "/root/Bitcoin-Graph-Explorer/src/test/resources/test.sh" ! ProcessLogger(_ => ()) 
+  def resetRegtest = "/root/Bitcoin-Graph-Explorer/src/test/docker/reset.sh" ! ProcessLogger(_ => ()) 
   // fixme: to use an script is dirty but i need to restart some containers...
 
-
   "Populate" should "safe 5 blocks as the blockchain contains 5 blocks" in {
     resetRegtest
     gen(5)

From 340078bfb0b566f350d5a609947c6fcbd44f1330 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 08:44:55 +0200
Subject: [PATCH 32/69] gitignore

---
 .gitignore | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.gitignore b/.gitignore
index 6252cb2..0c25d92 100755
--- a/.gitignore
+++ b/.gitignore
@@ -4,13 +4,9 @@ project/target/
 project/plugins/
 .idea/
 *.log
-neodb/
 .ensime
 .ensime_cache/
 src/main/resources/application.conf
 .history
 blockchain
-src/test/data/blockchain/*
-src/test/data/psql/*
-src/test/data/bitcoin/*
-
+src/test/data/*/*

From 250d12002e931eb12cabd2b78c2ec85dbb8211b5 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 08:52:42 +0200
Subject: [PATCH 33/69] add test data default folder structure

---
 .gitignore                       | 1 -
 src/test/data/bitcoin/.gitignore | 4 ++++
 src/test/data/psql/.gitignore    | 4 ++++
 3 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 src/test/data/bitcoin/.gitignore
 create mode 100644 src/test/data/psql/.gitignore

diff --git a/.gitignore b/.gitignore
index 0c25d92..d802fd8 100755
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,3 @@ project/plugins/
 src/main/resources/application.conf
 .history
 blockchain
-src/test/data/*/*
diff --git a/src/test/data/bitcoin/.gitignore b/src/test/data/bitcoin/.gitignore
new file mode 100644
index 0000000..5e7d273
--- /dev/null
+++ b/src/test/data/bitcoin/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore
diff --git a/src/test/data/psql/.gitignore b/src/test/data/psql/.gitignore
new file mode 100644
index 0000000..5e7d273
--- /dev/null
+++ b/src/test/data/psql/.gitignore
@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

From 10eb9ea60d25f1cd7f9b5315055f1c6346c34be4 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 09:08:04 +0200
Subject: [PATCH 34/69] change folders

---
 src/test/data/{bitcoin => }/.gitignore |  0
 src/test/data/psql/.gitignore          |  4 ----
 src/test/docker/reset.sh               | 16 ++++------------
 3 files changed, 4 insertions(+), 16 deletions(-)
 rename src/test/data/{bitcoin => }/.gitignore (100%)
 delete mode 100644 src/test/data/psql/.gitignore

diff --git a/src/test/data/bitcoin/.gitignore b/src/test/data/.gitignore
similarity index 100%
rename from src/test/data/bitcoin/.gitignore
rename to src/test/data/.gitignore
diff --git a/src/test/data/psql/.gitignore b/src/test/data/psql/.gitignore
deleted file mode 100644
index 5e7d273..0000000
--- a/src/test/data/psql/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore
diff --git a/src/test/docker/reset.sh b/src/test/docker/reset.sh
index f15dfab..7982132 100755
--- a/src/test/docker/reset.sh
+++ b/src/test/docker/reset.sh
@@ -1,16 +1,8 @@
-rm /root/Bitcoin-Graph-Explorer/src/test/data/bitcoin/regtest/ -rf
+mkdir -p /root/Bitcoin-Graph-Explorer/src/test/data/psql
+mkdir -p /root/Bitcoin-Graph-Explorer/src/test/data/bitcoin
+rm /root/Bitcoin-Graph-Explorer/src/test/data/bitcoin/ -rf
 rm /root/Bitcoin-Graph-Explorer/src/test/data/blockchain/ -rf
 cd /root/Bitcoin-Graph-Explorer/src/test/docker
 make stop start
 echo "waiting for components to be ready"
-echo "5"
-sleep 1s
-echo "4"
-sleep 1s
-echo "3"
-sleep 1s
-echo "2"
-sleep 1s
-echo "1"
-sleep 1s
-echo "done sleeping"
+sleep 5s

From 539fe268ac7dc99be0bcd5fe0c9aba03b722e264 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 11:21:49 +0200
Subject: [PATCH 35/69] fix postgres docker init and add proper waiting
 conditions

---
 src/test/docker/docker-compose.yml | 11 +++++----
 src/test/docker/reset.sh           | 38 +++++++++++++++++++++++-------
 2 files changed, 37 insertions(+), 12 deletions(-)

diff --git a/src/test/docker/docker-compose.yml b/src/test/docker/docker-compose.yml
index 0341fa9..4f0d312 100644
--- a/src/test/docker/docker-compose.yml
+++ b/src/test/docker/docker-compose.yml
@@ -17,8 +17,8 @@ services:
       -rpcuser=foo
       -server
     volumes:
-      - "/root/Bitcoin-Graph-Explorer/src/test/data/bitcoin:/home/bitcoin/.bitcoin"
-      - "$FOLDER/bitcoin/bitcoin.conf:/etc/bitcoin/bitcoin.conf"
+      - "${DIR}/data/bitcoin:/home/bitcoin/.bitcoin"
+      - "${DIR}/docker/bitcoin/bitcoin.conf:/etc/bitcoin/bitcoin.conf"
   postgres:
     image: postgres:10.1
     networks:
@@ -33,8 +33,11 @@ services:
       POSTGRES_PASSWORD: trivial
       POSTGRES_DB: postgres2
     volumes:
-      - "/root/Bitcoin-Graph-Explorer/src/test/data/psql:/var/lib/postgresql/data"
-      - "$FOLDER/postgres/postgres.conf:/var/lib/postgresql/data/postgresql.conf"
+      - "${DIR}/data/psql:/var/lib/postgresql/data"
+      - "${DIR}/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf"
+    command:
+      -c
+      config_file=/etc/postgresql/postgresql.conf
 networks:
   bge_network:
     driver: bridge
diff --git a/src/test/docker/reset.sh b/src/test/docker/reset.sh
index 7982132..7199e41 100755
--- a/src/test/docker/reset.sh
+++ b/src/test/docker/reset.sh
@@ -1,8 +1,30 @@
-mkdir -p /root/Bitcoin-Graph-Explorer/src/test/data/psql
-mkdir -p /root/Bitcoin-Graph-Explorer/src/test/data/bitcoin
-rm /root/Bitcoin-Graph-Explorer/src/test/data/bitcoin/ -rf
-rm /root/Bitcoin-Graph-Explorer/src/test/data/blockchain/ -rf
-cd /root/Bitcoin-Graph-Explorer/src/test/docker
-make stop start
-echo "waiting for components to be ready"
-sleep 5s
+#!/bin/bash
+export VERSION=SNAPSHOT
+export REPOSITORY=jorgemartinezpizarro
+export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+echo $DIR
+
+mkdir -p "${DIR}/data/psql"
+mkdir -p "${DIR}/data/bitcoin"
+rm "${DIR}/data/bitcoin/" -rf
+rm "${DIR}/data/blockchain/" -rf
+cd "${DIR}/docker"
+docker-compose kill
+docker-compose up -d
+## Wait until both docker images are ready
+for i in $(seq 1 25)
+do
+  docker exec -it docker_postgres_1 pg_isready -p 5433 > /dev/null 2> /dev/null
+  rc1=$?
+  docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333 getnetworkinfo > /dev/null 2> /dev/null
+  rc2=$?
+  if [ $rc1 -eq 0 ] && [ $rc2 -eq 0 ]
+  then
+    echo "start"
+    exit 0
+  else 
+    echo "waiting"
+  fi
+done
+echo "Timeout"
+exit 1

From 88fb1df6daaf5829cdb4213febcf7ec73784b321 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 11:24:01 +0200
Subject: [PATCH 36/69] remove unused file

---
 src/test/docker/Makefile | 32 --------------------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 src/test/docker/Makefile

diff --git a/src/test/docker/Makefile b/src/test/docker/Makefile
deleted file mode 100644
index f268426..0000000
--- a/src/test/docker/Makefile
+++ /dev/null
@@ -1,32 +0,0 @@
-export VERSION=SNAPSHOT
-export REPOSITORY=jorgemartinezpizarro
-export FOLDER=/root/Bitcoin-Graph-Explorer/src/test/docker
-## start
-start:
-	docker-compose up -d
-
-## stop
-stop:
-	docker-compose kill
-
-## clean
-clean:
-	docker-compose rm
-
-##logs
-logs:
-	docker-compose logs --tail="all" --follow bitcoin postgres
-
-## show this help screen
-help:
-	@printf "Available targets\n\n"
-	@awk '/^[a-zA-Z\-\_0-9]+:/ { \
-		helpMessage = match(lastLine, /^## (.*)/); \
-		if (helpMessage) { \
-			helpCommand = substr($$1, 0, index($$1, ":")); \
-			helpMessage = substr(lastLine, RSTART + 3, RLENGTH); \
-			printf "%-15s %s\n", helpCommand, helpMessage; \
-		} \
-	} \
-	{ lastLine = $$0 }' $(MAKEFILE_LIST)
-

From 51c8193c549414019ddb3d0558ab77fc92f19622 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 13:26:23 +0200
Subject: [PATCH 37/69] final version of testing

---
 src/main/scala/db/BitcoinDB.scala      |  1 +
 src/test/docker/docker-compose.yml     |  4 ++--
 src/test/docker/{reset.sh => start.sh} | 28 ++++++++++++++------------
 src/test/docker/stop.sh                | 12 +++++++++++
 src/test/{data => files}/.gitignore    |  0
 src/test/resources/application.conf    |  2 +-
 src/test/resources/reference.conf      | 21 +++++++++++++++++++
 src/test/scala/BGESpec.scala           | 24 +++++++++++++---------
 8 files changed, 67 insertions(+), 25 deletions(-)
 rename src/test/docker/{reset.sh => start.sh} (54%)
 create mode 100755 src/test/docker/stop.sh
 rename src/test/{data => files}/.gitignore (100%)
 create mode 100644 src/test/resources/reference.conf

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index be62761..6edd1bb 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -27,6 +27,7 @@ trait BitcoinDB {
   def DBNAME = conf.getString("databaseName")
   def URL = "jdbc:postgresql://" + HOST + "/" + DBNAME + OPTIONS
   def DRIVER = "org.postgresql.Driver"
+  println(URL)
 
   def statsDone: Boolean = {
     DB.withSession{
diff --git a/src/test/docker/docker-compose.yml b/src/test/docker/docker-compose.yml
index 4f0d312..6f5a6ed 100644
--- a/src/test/docker/docker-compose.yml
+++ b/src/test/docker/docker-compose.yml
@@ -17,7 +17,7 @@ services:
       -rpcuser=foo
       -server
     volumes:
-      - "${DIR}/data/bitcoin:/home/bitcoin/.bitcoin"
+      - "${DIR}/files/bitcoin:/home/bitcoin/.bitcoin"
       - "${DIR}/docker/bitcoin/bitcoin.conf:/etc/bitcoin/bitcoin.conf"
   postgres:
     image: postgres:10.1
@@ -33,7 +33,7 @@ services:
       POSTGRES_PASSWORD: trivial
       POSTGRES_DB: postgres2
     volumes:
-      - "${DIR}/data/psql:/var/lib/postgresql/data"
+      - "${DIR}/files/psql:/var/lib/postgresql/data"
       - "${DIR}/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf"
     command:
       -c
diff --git a/src/test/docker/reset.sh b/src/test/docker/start.sh
similarity index 54%
rename from src/test/docker/reset.sh
rename to src/test/docker/start.sh
index 7199e41..8c3c805 100755
--- a/src/test/docker/reset.sh
+++ b/src/test/docker/start.sh
@@ -2,28 +2,30 @@
 export VERSION=SNAPSHOT
 export REPOSITORY=jorgemartinezpizarro
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
-echo $DIR
 
-mkdir -p "${DIR}/data/psql"
-mkdir -p "${DIR}/data/bitcoin"
-rm "${DIR}/data/bitcoin/" -rf
+mkdir -p "${DIR}/files/psql"
+mkdir -p "${DIR}/files/bitcoin"
+rm "${DIR}/files/psql/" -rf
+rm "${DIR}/files/bitcoin/" -rf
 rm "${DIR}/data/blockchain/" -rf
 cd "${DIR}/docker"
+
 docker-compose kill
 docker-compose up -d
-## Wait until both docker images are ready
-for i in $(seq 1 25)
+
+echo "Waiting for docker to be ready"
+
+for i in $(seq 1 1000)
 do
-  docker exec -it docker_postgres_1 pg_isready -p 5433 > /dev/null 2> /dev/null
-  rc1=$?
+  docker exec docker_postgres_1 pg_isready -p 5433 > /dev/null 2> /dev/null
+  code1=$?
   docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333 getnetworkinfo > /dev/null 2> /dev/null
-  rc2=$?
-  if [ $rc1 -eq 0 ] && [ $rc2 -eq 0 ]
+  code2=$?
+  if [ $code1 -eq 0 ] && [ $code2 -eq 0 ]
   then
-    echo "start"
+    echo "Finished loading"
+    sleep 1s
     exit 0
-  else 
-    echo "waiting"
   fi
 done
 echo "Timeout"
diff --git a/src/test/docker/stop.sh b/src/test/docker/stop.sh
new file mode 100755
index 0000000..def34f8
--- /dev/null
+++ b/src/test/docker/stop.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+export VERSION=SNAPSHOT
+export REPOSITORY=jorgemartinezpizarro
+export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
+echo $DIR
+
+mkdir -p "${DIR}/data/psql"
+mkdir -p "${DIR}/data/bitcoin"
+rm "${DIR}/data/bitcoin/" -rf
+rm "${DIR}/data/blockchain/" -rf
+cd "${DIR}/docker"
+docker-compose kill
diff --git a/src/test/data/.gitignore b/src/test/files/.gitignore
similarity index 100%
rename from src/test/data/.gitignore
rename to src/test/files/.gitignore
diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf
index 0f1cfde..98872d3 100644
--- a/src/test/resources/application.conf
+++ b/src/test/resources/application.conf
@@ -3,7 +3,7 @@ closureReadSize=1000 // blocks
 closureTransactionSize=50000
 balanceTransactionSize=50000
 richlistSize=1000
-dataDir = "/root/Bitcoin-Graph-Explorer/src/test/data/blockchain/"
+dataDir = "/root/Bitcoin-Graph-Explorer/src/test/files/blockchain/"
 levelDBFile = ${dataDir}"level.db"
 lockFile = ${dataDir}"lock"
 dustLimit=100000
diff --git a/src/test/resources/reference.conf b/src/test/resources/reference.conf
new file mode 100644
index 0000000..98872d3
--- /dev/null
+++ b/src/test/resources/reference.conf
@@ -0,0 +1,21 @@
+populateTransactionSize=100000 // movemements
+closureReadSize=1000 // blocks
+closureTransactionSize=50000
+balanceTransactionSize=50000
+richlistSize=1000
+dataDir = "/root/Bitcoin-Graph-Explorer/src/test/files/blockchain/"
+levelDBFile = ${dataDir}"level.db"
+lockFile = ${dataDir}"lock"
+dustLimit=100000
+databaseName="postgres2"
+username = "root"
+password = "trivial"
+host="localhost:5433"
+jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
+mdb.path = ${dataDir}"mdb/"
+mdb.transactionSize = 100000
+network=regtest # accepts main, testnet, regtest
+resumeBlockSize=10000000 # depending on the available ram
+bitcoin.ip=localhost
+bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
+balanceUpdateLimit=70000 
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index 8ad89b3..d6f6944 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -5,22 +5,28 @@ import org.scalatest._
 class BGESpec extends FlatSpec with Matchers {
   val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
   // fixme: to use an script is dirty but we need to access to bitcoincli which lives in a container
-  def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
-  def resetRegtest = "/root/Bitcoin-Graph-Explorer/src/test/docker/reset.sh" ! ProcessLogger(_ => ()) 
+  def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! //ProcessLogger(_ => ())
+
+  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! //ProcessLogger(_ => ()) 
+
+  def stopDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/stop.sh" ! //ProcessLogger(_ => ())
+
   // fixme: to use an script is dirty but i need to restart some containers...
+  // for some reason starting bge 2 times in same process fails... maybe cause singleton pattern?
 
-  "Populate" should "safe 5 blocks as the blockchain contains 5 blocks" in {
-    resetRegtest
-    gen(5)
+  "Populate" should "save 10 blocks as the blockchain contains 10 blocks" in {
+    startDocker
+    gen(10)
     Explorer.populate
-    Explorer.blockCount should be (6)
+    Explorer.blockCount should be (11)
   }
 
-  "Resume" should "add another 10 blocks after being added to blockchain" in {
+  "Resume" should "after populate 5 blocks and add 5 more, resume should add 5 blocks more" in {
     gen(10)
     Explorer.resume
-    Explorer.blockCount should be (16)
-    
+    val result = Explorer.blockCount
+//    stopDocker
+    result should be (21)
   }
 }
 

From f489f6c28e6ba7f63adabfa199ca930c4f1a3f1f Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 14:15:05 +0200
Subject: [PATCH 38/69] use ports forward to avoid postgres docker collisions
 ...

---
 src/main/resources/reference.conf  | 8 ++++----
 src/test/docker/docker-compose.yml | 8 ++++----
 src/test/docker/start.sh           | 5 ++---
 src/test/scala/BGESpec.scala       | 4 ++--
 4 files changed, 12 insertions(+), 13 deletions(-)

diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index 16416f2..f005bd1 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -11,15 +11,15 @@ levelDBFile = ${dataDir}"level.db"
 lockFile = ${dataDir}"lock"
 dustLimit=546
 databaseName="postgres"
-username = "root"
-password = "trivial"
-host = "localhost"
+username = "user"
+password = "pass"
+host = "localhost:1234"
 jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
 mdb.path = ${dataDir}"mdb/"
 mdb.transactionSize = 100000
 network=main # accepts main, testnet, regtest
 logger.scala.slick=ERROR
-resumeBlockSize=500 # depending on the available ram
+resumeBlockSize=1000000 # depending on the available ram
 bitcoin.ip=localhost
 bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
 balanceUpdateLimit=50000 # for balance changes beyond this size, create new balance tables
diff --git a/src/test/docker/docker-compose.yml b/src/test/docker/docker-compose.yml
index 6f5a6ed..b95f2bf 100644
--- a/src/test/docker/docker-compose.yml
+++ b/src/test/docker/docker-compose.yml
@@ -17,8 +17,8 @@ services:
       -rpcuser=foo
       -server
     volumes:
-      - "${DIR}/files/bitcoin:/home/bitcoin/.bitcoin"
-      - "${DIR}/docker/bitcoin/bitcoin.conf:/etc/bitcoin/bitcoin.conf"
+      - "$DIR/files/bitcoin:/home/bitcoin/.bitcoin"
+      - "$DIR/docker/bitcoin/bitcoin.conf:/etc/bitcoin/bitcoin.conf"
   postgres:
     image: postgres:10.1
     networks:
@@ -33,8 +33,8 @@ services:
       POSTGRES_PASSWORD: trivial
       POSTGRES_DB: postgres2
     volumes:
-      - "${DIR}/files/psql:/var/lib/postgresql/data"
-      - "${DIR}/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf"
+      - "$DIR/files/psql:/var/lib/postgresql/data"
+      - "$DIR/docker/postgres/postgres.conf:/etc/postgresql/postgresql.conf"
     command:
       -c
       config_file=/etc/postgresql/postgresql.conf
diff --git a/src/test/docker/start.sh b/src/test/docker/start.sh
index 8c3c805..b80a59f 100755
--- a/src/test/docker/start.sh
+++ b/src/test/docker/start.sh
@@ -3,11 +3,11 @@ export VERSION=SNAPSHOT
 export REPOSITORY=jorgemartinezpizarro
 export DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
 
-mkdir -p "${DIR}/files/psql"
+#mkdir -p "${DIR}/files/psql"
 mkdir -p "${DIR}/files/bitcoin"
 rm "${DIR}/files/psql/" -rf
 rm "${DIR}/files/bitcoin/" -rf
-rm "${DIR}/data/blockchain/" -rf
+rm "${DIR}/files/blockchain/" -rf
 cd "${DIR}/docker"
 
 docker-compose kill
@@ -24,7 +24,6 @@ do
   if [ $code1 -eq 0 ] && [ $code2 -eq 0 ]
   then
     echo "Finished loading"
-    sleep 1s
     exit 0
   fi
 done
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index d6f6944..6f5f4ff 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -21,11 +21,11 @@ class BGESpec extends FlatSpec with Matchers {
     Explorer.blockCount should be (11)
   }
 
-  "Resume" should "after populate 5 blocks and add 5 more, resume should add 5 blocks more" in {
+  "Resume" should "after add 10 blocks, resume save the 10 blocks" in {
     gen(10)
     Explorer.resume
     val result = Explorer.blockCount
-//    stopDocker
+    stopDocker
     result should be (21)
   }
 }

From 77e300dd2dde4316382faabe3a134bbbbaa5bf90 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 20 May 2018 14:21:35 +0200
Subject: [PATCH 39/69] remove redundant log - debug

---
 src/main/scala/db/BitcoinDB.scala |  1 -
 src/test/scala/BGESpec.scala      | 13 +++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 6edd1bb..be62761 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -27,7 +27,6 @@ trait BitcoinDB {
   def DBNAME = conf.getString("databaseName")
   def URL = "jdbc:postgresql://" + HOST + "/" + DBNAME + OPTIONS
   def DRIVER = "org.postgresql.Driver"
-  println(URL)
 
   def statsDone: Boolean = {
     DB.withSession{
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index 6f5f4ff..1466164 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -3,15 +3,17 @@ import util._
 import org.scalatest._
 
 class BGESpec extends FlatSpec with Matchers {
+  
   val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
+  
   // fixme: to use an script is dirty but we need to access to bitcoincli which lives in a container
-  def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! //ProcessLogger(_ => ())
+  
+  def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
 
-  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! //ProcessLogger(_ => ()) 
+  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! ProcessLogger(_ => ()) 
 
-  def stopDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/stop.sh" ! //ProcessLogger(_ => ())
+  def stopDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/stop.sh" ! ProcessLogger(_ => ())
 
-  // fixme: to use an script is dirty but i need to restart some containers...
   // for some reason starting bge 2 times in same process fails... maybe cause singleton pattern?
 
   "Populate" should "save 10 blocks as the blockchain contains 10 blocks" in {
@@ -19,9 +21,12 @@ class BGESpec extends FlatSpec with Matchers {
     gen(10)
     Explorer.populate
     Explorer.blockCount should be (11)
+    // stopDocker
   }
 
   "Resume" should "after add 10 blocks, resume save the 10 blocks" in {
+    // for some reason starting bge 2 times in same process fails... maybe cause we use Objects?
+    // startDocker
     gen(10)
     Explorer.resume
     val result = Explorer.blockCount

From bff2ec0da88dc615f6711ea4a904cd36459ffeb6 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 27 May 2018 16:14:38 +0200
Subject: [PATCH 40/69] convert updateBalances variables to mutable maps.
 Rewrite closure balance updates. Add asserts to be testable with the current
 tests. Add readUTXOsSize parameter. increase java runtime memory limits.
 Remove application.conf from test/resources.

---
 build.sbt                           |  4 +-
 src/main/resources/reference.conf   |  1 +
 src/main/scala/bge/Explorer.scala   |  8 +--
 src/main/scala/db/BitcoinDB.scala   | 81 +++++++++++++++++++++--------
 src/main/scala/util/package.scala   |  1 +
 src/test/resources/application.conf | 21 --------
 src/test/resources/reference.conf   |  1 +
 src/test/scala/BGESpec.scala        | 39 ++++++++------
 8 files changed, 89 insertions(+), 67 deletions(-)
 delete mode 100644 src/test/resources/application.conf

diff --git a/build.sbt b/build.sbt
index 6958156..a353241 100644
--- a/build.sbt
+++ b/build.sbt
@@ -92,7 +92,7 @@ scalacOptions ++= Seq(
 //    ,"-Xlog-implicit-conversions"
 )
 
-javaOptions in run += "-Xmx16G"
-javaOptions in run += "-Xms1G"
+javaOptions in run += "-Xmx32G"
+javaOptions in run += "-Xms16"
 
 fork := true
diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf
index f005bd1..1819993 100755
--- a/src/main/resources/reference.conf
+++ b/src/main/resources/reference.conf
@@ -23,3 +23,4 @@ resumeBlockSize=1000000 # depending on the available ram
 bitcoin.ip=localhost
 bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
 balanceUpdateLimit=50000 # for balance changes beyond this size, create new balance tables
+checkUTXOsSize=100000
\ No newline at end of file
diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index f64abbb..ed8311b 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -17,7 +17,7 @@ object Explorer extends App with db.BitcoinDB {
       if (!statsDone || rest.headOption == Some("--force"))
         populate
       Seq("touch",lockFile).!
-      iterateResume(true)
+      iterateResume()
 
     case "populate"::rest             =>
 
@@ -42,7 +42,7 @@ object Explorer extends App with db.BitcoinDB {
 
         Available commands:
 
-         start [--force]: populate if necessary or --force, then resume --newstats
+         start [--force]: populate if necessary or --force, then resume
          stop: ask bge gently to stop at the end of the current iteration. Note that thís can take a few days if you are in the populate phase.
          populate: create the database movements with movements and closures.  Deletes any existing data.
          resume [--newstats]: update the database generated by populate with new incoming data. Works only if populate is already done. The newstats flag forces a regeneration of balance, richlist and stats tables.
@@ -75,7 +75,7 @@ object Explorer extends App with db.BitcoinDB {
     lazy val outputMap: UTXOs = new UTXOs (table)
     // (txhash,index) -> (address,value,blockIn)
     val values = for ( (_,(_,value,_)) <- outputMap.view) yield value //makes it a lazy collection
-    val tuple = values.grouped(100000).foldLeft((0,0L)){
+    val tuple = values.grouped(checkUTXOsSize).foldLeft((0,0L)){
       case ((count,sum),group) =>
         log.info(count + " elements read at ")
         val seq = group.toSeq
@@ -177,7 +177,7 @@ object Explorer extends App with db.BitcoinDB {
 
     if (changedAddresses.size < balanceUpdateLimit )
     {
-      updateBalanceTables(changedAddresses, changedReps)
+      updateBalanceTables(changedAddresses.toMap, changedReps)
       insertRichestAddresses
       insertRichestClosures
       updateStatistics(changedReps,addedAds, addedReps)
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index be62761..3f6bb9e 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -111,7 +111,7 @@ trait BitcoinDB {
       // '\x' is not an address
       Q.updateNA("""insert into balances select address, sum(value) as balance from utxo where address != '\x' group by address;""").execute
 
-     (Q.u + "create index balance on balances(balance)").execute
+      (Q.u + "create index balance on balances(balance)").execute
 
       Q.updateNA("""
          insert into closure_balances
@@ -137,14 +137,14 @@ trait BitcoinDB {
     }
   }
 
-  def updateBalanceTables(changedAddresses: Map[Hash, Long], changedReps: Map[Hash, Set[Hash]]) = {
+  def updateBalanceTables(changedAddresses: scala.collection.immutable.Map[Hash, Long], changedReps: Map[Hash, Set[Hash]]) = {
     var clock = System.currentTimeMillis
     log.info("Updating balances ...")
     currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
 
     DB withSession { implicit session =>
 
-      def updateAdsBalancesTable[A <: Table[(Array[Byte], Long)] with AddressField](adsAndBalances: Map[Hash, Long], balances: TableQuery[A]): Unit = {
+      def updateAdsBalancesTable[A <: Table[(Array[Byte], Long)] with AddressField](adsAndBalances: scala.collection.immutable.Map[Hash, Long], balances: TableQuery[A]): Unit = {
         for {
           (address, balance) <- adsAndBalances
           addressArray = Hash.hashToArray(address)
@@ -156,49 +156,84 @@ trait BitcoinDB {
       }
       // Hash.zero(0) is not an address
       session.withTransaction {
-        val adsAndBalances = for ((address, change) <- changedAddresses - Hash.zero(0))
+
+        // Check that the balances are consistent. Still missing check that all representants are actually representants.
+        val b3 = balances.map(_.balance).sum.run.getOrElse(0)
+        val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
+
+        assert(b3 == b4, s"$b3 is not equal $b4. Error updating balances!")
+
+        val adsAndBalances: scala.collection.immutable.Map[Hash, Long] = for ((address, change) <- changedAddresses - Hash.zero(0))
                              yield (address,
                                     balances.filter(_.address === Hash.hashToArray(address)).
                                       map(_.balance).firstOption.getOrElse(0L) + change)
 
         updateAdsBalancesTable(adsAndBalances, balances)
-
-        val table = LmdbMap.open("closures")
-        val unionFindTable = new ClosureMap(table)
-        val closures = new DisjointSets[Hash](unionFindTable)
-
+        // TODO Dirty method: write if functionally
         val repsAndChanges: collection.mutable.Map[Hash, Long] = collection.mutable.Map()
-
-        for ((address, change) <- changedAddresses - Hash.zero(0) ) {
-          val rep = closures.find(address)._1.getOrElse(address)
-          val newBalance = repsAndChanges.getOrElse(rep, 0L) + change
-          repsAndChanges += (rep -> newBalance)
+        for ((rep,_) <- changedReps) {
+          repsAndChanges += (rep -> 0L)
+        }
+        for ((address, _) <- changedAddresses - Hash.zero(0) ) {
+
+          val representant = changedReps.filter( x => x._2 contains address).toList match {
+            case Nil =>
+              address
+            case m: List[(Hash, Set[Hash])] =>
+              m.head._1
+          }
+          repsAndChanges += (representant -> 0L)
         }
 
-        // don't forget the new reps that had no balance change!
-        for ((rep,_) <- changedReps)
-          repsAndChanges += (rep -> repsAndChanges.getOrElse(rep, 0L))
-  
-        val repsAndBalances: Map[Hash, Long] =
+        for ((representant, _) <- repsAndChanges) {
+          val oldReps = changedReps.getOrElse(representant, Set())+representant
+          val change = changedAddresses.filter (x => oldReps contains x._1 ).map(_._2).sum
+          println(s"$representant: $change")
+          repsAndChanges += (representant -> change)
+        }
+        ///////////////////////////////////////////////////// 
+        val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
           for {
-            (rep, change) <- repsAndChanges
+            (rep, change) <- repsAndChanges.toMap
             oldReps = (changedReps.getOrElse(rep, Set()) + rep).map(Hash.hashToArray(_))
           } yield (rep,
                    closureBalances.filter(_.address inSetBind oldReps).
                      map(_.balance).sum.run.getOrElse(0L) + change)
 
+        // Check that balances are at least positive. Seems naive but it find several errors
+        for ((rep, balance) <- repsAndBalances) {
+          val oldReps: Set[Hash] =
+            if (balance < 0)
+              changedReps.getOrElse(rep, Set())+rep
+            else
+              Set()
+          assert(balance >= 0, s"""$rep hat negative balance $balance
+          Old value: ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
+          Old reps: $oldReps
+          Diffs: ${changedAddresses.map(_._2).sum} should be ${repsAndChanges.map(_._2).sum}
+          Changed: ${repsAndChanges.getOrElse(rep, 0)}
+          New in closure: ${changedAddresses.filter( x => oldReps contains x._1 )}""")
+        }
+
+        assert(changedAddresses.map(_._2).sum == repsAndChanges.map(_._2).sum, s"""
+            Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
+            Adds: $changedAddresses
+            Reps: $repsAndChanges
+        """)
         updateAdsBalancesTable(repsAndBalances, closureBalances)
 
         // delete all oldReps that have been unified into new ones
         val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
-        closureBalances.filter(_.address inSet toDelete).delete
+        closureBalances.filter(_.address inSetBind toDelete).delete
 
-        table.close
+        // Check that the balances are consistent. Still missing check that all representants are actually representants.
+        val b1 = balances.map(_.balance).sum.run.getOrElse(0)
+        val b2 = closureBalances.map(_.balance).sum.run.getOrElse(0)
+        assert(b1 == b2, s"$b1 is not equal $b2. Error updating balances!")
 
         log.info("%s balances updated in %s s, %s µs per address "
                    format
                    (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
-
       }
     }
   }
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index df621ad..edda35a 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -28,6 +28,7 @@ package object util
   lazy val balanceUpdateLimit = conf.getInt("balanceUpdateLimit")
   lazy val blockStoreFile = new java.io.File(conf.getString("levelDBFile"))
   lazy val lockFile = conf.getString("lockFile")
+  lazy val checkUTXOsSize = conf.getInt("checkUTXOsSize")
   lazy val internetAddress = conf.getString("bitcoin.ip") match {
     case "localhost" => 
       InetAddress.getLocalHost()
diff --git a/src/test/resources/application.conf b/src/test/resources/application.conf
deleted file mode 100644
index 98872d3..0000000
--- a/src/test/resources/application.conf
+++ /dev/null
@@ -1,21 +0,0 @@
-populateTransactionSize=100000 // movemements
-closureReadSize=1000 // blocks
-closureTransactionSize=50000
-balanceTransactionSize=50000
-richlistSize=1000
-dataDir = "/root/Bitcoin-Graph-Explorer/src/test/files/blockchain/"
-levelDBFile = ${dataDir}"level.db"
-lockFile = ${dataDir}"lock"
-dustLimit=100000
-databaseName="postgres2"
-username = "root"
-password = "trivial"
-host="localhost:5433"
-jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
-mdb.path = ${dataDir}"mdb/"
-mdb.transactionSize = 100000
-network=regtest # accepts main, testnet, regtest
-resumeBlockSize=10000000 # depending on the available ram
-bitcoin.ip=localhost
-bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
-balanceUpdateLimit=70000 
diff --git a/src/test/resources/reference.conf b/src/test/resources/reference.conf
index 98872d3..5d3bf4d 100644
--- a/src/test/resources/reference.conf
+++ b/src/test/resources/reference.conf
@@ -19,3 +19,4 @@ resumeBlockSize=10000000 # depending on the available ram
 bitcoin.ip=localhost
 bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
 balanceUpdateLimit=70000 
+checkUTXOsSize=1000000
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index 1466164..f38602d 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -3,35 +3,40 @@ import util._
 import org.scalatest._
 
 class BGESpec extends FlatSpec with Matchers {
-  
+
+  // FIXME: Use bitcoin rpc client and docker tools instead 
+
   val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
-  
-  // fixme: to use an script is dirty but we need to access to bitcoincli which lives in a container
-  
+
   def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
 
-  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! ProcessLogger(_ => ()) 
+  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! ProcessLogger(_ => ())
 
   def stopDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/stop.sh" ! ProcessLogger(_ => ())
 
-  // for some reason starting bge 2 times in same process fails... maybe cause singleton pattern?
+  def addTxs = "/root/Bitcoin-Graph-Explorer/src/test/docker/create.sh" ! ProcessLogger(_ => ())
 
   "Populate" should "save 10 blocks as the blockchain contains 10 blocks" in {
-    startDocker
-    gen(10)
+    startDocker 
+    gen(1)
+    gen(100)
     Explorer.populate
-    Explorer.blockCount should be (11)
-    // stopDocker
-  }
+    Explorer.blockCount should be (102)
+   }
 
   "Resume" should "after add 10 blocks, resume save the 10 blocks" in {
-    // for some reason starting bge 2 times in same process fails... maybe cause we use Objects?
-    // startDocker
-    gen(10)
+    gen(1)
+    gen(1)
+    Explorer.resume
+    Explorer.blockCount should be (104)
+    gen(100)
+    Explorer.resume
+    addTxs
     Explorer.resume
-    val result = Explorer.blockCount
-    stopDocker
-    result should be (21)
+    Explorer.blockCount should be (204)
+    // Test config?
+    //stopDocker // Deactivate it only if you wannt to analize the database
+    true
   }
 }
 

From 3d6e70aefa4f03af755757bcd4943ba4d11a6499 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sun, 27 May 2018 21:44:16 +0200
Subject: [PATCH 41/69] improve asserts on balances. create first time if sums
 are wrong.

---
 src/main/scala/db/BitcoinDB.scala | 38 ++++++++++++++++++-------------
 src/test/scala/BGESpec.scala      |  2 +-
 2 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 3f6bb9e..72e2825 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -144,6 +144,15 @@ trait BitcoinDB {
 
     DB withSession { implicit session =>
 
+      val b3 = balances.map(_.balance).sum.run.getOrElse(0)
+      val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
+
+      if (b3 != b4) {
+        log.info("Incomplete or wrong balances, generating it from scratch")
+        createBalanceTables
+      }
+      else {
+
       def updateAdsBalancesTable[A <: Table[(Array[Byte], Long)] with AddressField](adsAndBalances: scala.collection.immutable.Map[Hash, Long], balances: TableQuery[A]): Unit = {
         for {
           (address, balance) <- adsAndBalances
@@ -158,11 +167,6 @@ trait BitcoinDB {
       session.withTransaction {
 
         // Check that the balances are consistent. Still missing check that all representants are actually representants.
-        val b3 = balances.map(_.balance).sum.run.getOrElse(0)
-        val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
-
-        assert(b3 == b4, s"$b3 is not equal $b4. Error updating balances!")
-
         val adsAndBalances: scala.collection.immutable.Map[Hash, Long] = for ((address, change) <- changedAddresses - Hash.zero(0))
                              yield (address,
                                     balances.filter(_.address === Hash.hashToArray(address)).
@@ -188,7 +192,6 @@ trait BitcoinDB {
         for ((representant, _) <- repsAndChanges) {
           val oldReps = changedReps.getOrElse(representant, Set())+representant
           val change = changedAddresses.filter (x => oldReps contains x._1 ).map(_._2).sum
-          println(s"$representant: $change")
           repsAndChanges += (representant -> change)
         }
         ///////////////////////////////////////////////////// 
@@ -200,25 +203,26 @@ trait BitcoinDB {
                    closureBalances.filter(_.address inSetBind oldReps).
                      map(_.balance).sum.run.getOrElse(0L) + change)
 
-        // Check that balances are at least positive. Seems naive but it find several errors
+        // Check that balances are at least positive. Seems naive but it catch lots of errors
         for ((rep, balance) <- repsAndBalances) {
           val oldReps: Set[Hash] =
             if (balance < 0)
               changedReps.getOrElse(rep, Set())+rep
             else
               Set()
-          assert(balance >= 0, s"""$rep hat negative balance $balance
-          Old value: ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
-          Old reps: $oldReps
-          Diffs: ${changedAddresses.map(_._2).sum} should be ${repsAndChanges.map(_._2).sum}
-          Changed: ${repsAndChanges.getOrElse(rep, 0)}
-          New in closure: ${changedAddresses.filter( x => oldReps contains x._1 )}""")
+          assert(balance >= 0, s"""
+            $rep hat negative balance $balance
+              Old value: ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
+              Old reps: ${oldReps.size} elements
+              Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
+              Changed: ${repsAndChanges.getOrElse(rep, 0)}
+              Representants/New reps: ${changedReps.map(_._2.size).sum}/${changedReps.size}
+          """)
         }
 
         assert(changedAddresses.map(_._2).sum == repsAndChanges.map(_._2).sum, s"""
-            Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
-            Adds: $changedAddresses
-            Reps: $repsAndChanges
+              Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
+              Representants/New reps: ${changedReps.map(_._2.size).sum}/${changedReps.size}
         """)
         updateAdsBalancesTable(repsAndBalances, closureBalances)
 
@@ -231,10 +235,12 @@ trait BitcoinDB {
         val b2 = closureBalances.map(_.balance).sum.run.getOrElse(0)
         assert(b1 == b2, s"$b1 is not equal $b2. Error updating balances!")
 
+      
         log.info("%s balances updated in %s s, %s µs per address "
                    format
                    (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
       }
+      }
     }
   }
 
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index f38602d..5e3ac59 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -35,7 +35,7 @@ class BGESpec extends FlatSpec with Matchers {
     Explorer.resume
     Explorer.blockCount should be (204)
     // Test config?
-    //stopDocker // Deactivate it only if you wannt to analize the database
+    //stopDocker // Deactivate it only if you want to analize the database after the tests ran
     true
   }
 }

From 0b9124d796eeb2fb63a928e02c02ed3cfc10caa9 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Mon, 28 May 2018 20:01:13 +0200
Subject: [PATCH 42/69] use DisjointSets for save resume partial tree instead
 of simulate it with the old Map oldReps.

---
 src/main/scala/bge/Explorer.scala             |   9 +-
 .../scala/bge/actions/ResumeClosure.scala     |  16 +-
 src/main/scala/bge/core/AddressClosure.scala  |   4 +
 src/main/scala/db/BitcoinDB.scala             | 168 +++++++-----------
 src/test/scala/BGESpec.scala                  |   5 +-
 5 files changed, 89 insertions(+), 113 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index ed8311b..8e1f09c 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -115,7 +115,14 @@ object Explorer extends App with db.BitcoinDB {
     val read = new ResumeBlockReader
     val closure = new ResumeClosure(read.processedBlocks)
     log.info("making new stats")
-    resumeStats(read.changedAddresses, closure.changedReps, closure.addedAds, closure.addedReps)
+    val t: Map[Hash, Set[Hash]] = Map.empty
+
+    for {
+      key <- closure.changedReps.elements.keys
+      parent = closure.changedReps.onlyFind(key) 
+    } { t += (parent -> (t.getOrElse(key, Set()) + key))}
+
+    resumeStats(read.changedAddresses, t, closure.addedAds, closure.addedReps)
   }
 
   def rollBackToLastStatIfNecessary: Unit =
diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index 884e23f..5b9cfcc 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -12,8 +12,7 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
   lazy val table = LmdbMap.open("closures")
   override lazy val unionFindTable = new ClosureMap(table)
 
-  lazy val changedReps = Map[Hash,Set[Hash]]()
-
+  
 
   override def insertInputsIntoTree(addressList: Iterable[Hash], tree: DisjointSets[Hash]): DisjointSets[Hash] =
   {
@@ -36,15 +35,8 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
     var addRepFlag = 1
 
     def recordChangedRep(newRep: Hash, oldRep: Hash) = {
-      // a changed to b, then b changed to c
-      // 1 we insert (b, a) in the map
-      // 2 we receive (c, b) cause b is already new, replace it with (c, {a,b})
-      val oldReps = changedReps.getOrElse(newRep,Set())
-      if (changedReps.contains(oldRep)) {
-        changedReps += (newRep -> (changedReps(oldRep)+ oldRep))
-        changedReps -= oldRep
-      }
-      else changedReps += (newRep -> (oldReps + oldRep) )
+      // we store in a DisjoinSets the partial changes
+      changedReps = changedReps.add(newRep).add(oldRep).union(Iterable(newRep, oldRep))
     }
 
 
@@ -64,11 +56,11 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
           case _ => addRepFlag = 0 // one of the elements had this rep before => don't count a new one
         }
     }
+
     addedReps += addRepFlag
 
     result
 
-
   }
 
 
diff --git a/src/main/scala/bge/core/AddressClosure.scala b/src/main/scala/bge/core/AddressClosure.scala
index a1a7845..f0e3173 100644
--- a/src/main/scala/bge/core/AddressClosure.scala
+++ b/src/main/scala/bge/core/AddressClosure.scala
@@ -18,11 +18,15 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
   var addedAds = 0
   var addedReps = 0
 
+  var temp = new ClosureMap(Map.empty)
+  var changedReps = new DisjointSets[Hash](temp)
   def saveTree(tree: DisjointSets[Hash]): Int
 
   def generateTree: DisjointSets[Hash] =
   {
     // TODO: Same problem as with BlockReader, there we used a function "pre" to initialize the values. 
+    temp = new ClosureMap(Map.empty)
+    changedReps = new DisjointSets[Hash](temp)
     addedAds = 0;
     addedReps = 0;
     def addBlocks(startIndex: Int, tree: DisjointSets[Hash]): DisjointSets[Hash] = {
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 72e2825..83f7d9c 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -137,13 +137,29 @@ trait BitcoinDB {
     }
   }
 
+  def updateAdsBalancesTable[A <: Table[(Array[Byte], Long)] with AddressField](adsAndBalances: scala.collection.immutable.Map[Hash, Long], balances: TableQuery[A]): Unit = {
+    DB withSession { implicit session =>
+      for {
+        (address, balance) <- adsAndBalances
+        addressArray = Hash.hashToArray(address)
+      }
+      if (balance != 0L)
+        balances.insertOrUpdate(addressArray, balance)
+      else
+        balances.filter(_.address === addressArray).delete
+    }
+  }
+
   def updateBalanceTables(changedAddresses: scala.collection.immutable.Map[Hash, Long], changedReps: Map[Hash, Set[Hash]]) = {
     var clock = System.currentTimeMillis
     log.info("Updating balances ...")
     currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
 
-    DB withSession { implicit session =>
+    lazy val table = LmdbMap.open("closures")
+    lazy val unionFindTable = new ClosureMap(table)
+    lazy val closures = new DisjointSets[Hash](unionFindTable)
 
+    DB withSession{ implicit session => 
       val b3 = balances.map(_.balance).sum.run.getOrElse(0)
       val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
 
@@ -152,94 +168,66 @@ trait BitcoinDB {
         createBalanceTables
       }
       else {
-
-      def updateAdsBalancesTable[A <: Table[(Array[Byte], Long)] with AddressField](adsAndBalances: scala.collection.immutable.Map[Hash, Long], balances: TableQuery[A]): Unit = {
-        for {
-          (address, balance) <- adsAndBalances
-          addressArray = Hash.hashToArray(address)
-        }
-        if (balance != 0L)
-          balances.insertOrUpdate(addressArray, balance)
-        else
-          balances.filter(_.address === addressArray).delete
-      }
-      // Hash.zero(0) is not an address
-      session.withTransaction {
-
-        // Check that the balances are consistent. Still missing check that all representants are actually representants.
-        val adsAndBalances: scala.collection.immutable.Map[Hash, Long] = for ((address, change) <- changedAddresses - Hash.zero(0))
+        session.withTransaction {
+          val adsAndBalances: scala.collection.immutable.Map[Hash, Long] = for ((address, change) <- changedAddresses - Hash.zero(0))
                              yield (address,
                                     balances.filter(_.address === Hash.hashToArray(address)).
                                       map(_.balance).firstOption.getOrElse(0L) + change)
 
-        updateAdsBalancesTable(adsAndBalances, balances)
-        // TODO Dirty method: write if functionally
-        val repsAndChanges: collection.mutable.Map[Hash, Long] = collection.mutable.Map()
-        for ((rep,_) <- changedReps) {
-          repsAndChanges += (rep -> 0L)
-        }
-        for ((address, _) <- changedAddresses - Hash.zero(0) ) {
-
-          val representant = changedReps.filter( x => x._2 contains address).toList match {
-            case Nil =>
-              address
-            case m: List[(Hash, Set[Hash])] =>
-              m.head._1
+          updateAdsBalancesTable(adsAndBalances, balances)
+
+          val repsAndChanges: collection.immutable.Map[Hash, Long] = 
+            (changedReps.map(_._1).toList ++ (changedAddresses - Hash.zero(0)).map(_._1).toList)
+              .distinct
+              .map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
+              .distinct
+              .map(r => {(r, changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum)})
+              .toMap
+
+          val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
+            (for {
+              (rep, change) <- repsAndChanges.toMap
+              representant = closures.find(rep)._1.getOrElse(rep)
+              oldies = (changedReps.getOrElse(rep, Set())+rep).map(Hash.hashToArray(_))
+            } yield (
+              representant,
+              closureBalances.filter(_.address inSetBind oldies).map(_.balance).sum.run.getOrElse(0L) + change
+            )).toMap
+
+          for ((rep, balance) <- repsAndBalances) {
+            val oldReps: Set[Hash] =
+              if (balance < 0)
+                changedReps.getOrElse(rep, Set())+rep
+              else
+                Set()
+            assert(balance >= 0, s"""
+              $rep hat negative balance $balance
+                Old value: ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
+                Old reps: ${oldReps.size} elements
+                Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
+                Changed: ${repsAndChanges.getOrElse(rep, 0)}
+                Representants/New reps: ${changedReps.map(_._2.size).sum}/${changedReps.size}
+            """)
           }
-          repsAndChanges += (representant -> 0L)
-        }
 
-        for ((representant, _) <- repsAndChanges) {
-          val oldReps = changedReps.getOrElse(representant, Set())+representant
-          val change = changedAddresses.filter (x => oldReps contains x._1 ).map(_._2).sum
-          repsAndChanges += (representant -> change)
-        }
-        ///////////////////////////////////////////////////// 
-        val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
-          for {
-            (rep, change) <- repsAndChanges.toMap
-            oldReps = (changedReps.getOrElse(rep, Set()) + rep).map(Hash.hashToArray(_))
-          } yield (rep,
-                   closureBalances.filter(_.address inSetBind oldReps).
-                     map(_.balance).sum.run.getOrElse(0L) + change)
-
-        // Check that balances are at least positive. Seems naive but it catch lots of errors
-        for ((rep, balance) <- repsAndBalances) {
-          val oldReps: Set[Hash] =
-            if (balance < 0)
-              changedReps.getOrElse(rep, Set())+rep
-            else
-              Set()
-          assert(balance >= 0, s"""
-            $rep hat negative balance $balance
-              Old value: ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
-              Old reps: ${oldReps.size} elements
-              Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
-              Changed: ${repsAndChanges.getOrElse(rep, 0)}
-              Representants/New reps: ${changedReps.map(_._2.size).sum}/${changedReps.size}
+          assert(changedAddresses.map(_._2).sum == repsAndChanges.map(_._2).sum, s"""
+                Address changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size})
+                  ${if (changedAddresses.size < 20) changedAddresses.toList.map(p => p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
+                Wallets changed ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
+                  ${if (repsAndChanges.size < 20) repsAndChanges.toList.map(p=> p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
+                Representants/New reps: ${changedReps.map(_._2.size).sum} ${changedReps.size}
           """)
-        }
 
-        assert(changedAddresses.map(_._2).sum == repsAndChanges.map(_._2).sum, s"""
-              Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
-              Representants/New reps: ${changedReps.map(_._2.size).sum}/${changedReps.size}
-        """)
-        updateAdsBalancesTable(repsAndBalances, closureBalances)
-
-        // delete all oldReps that have been unified into new ones
-        val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
-        closureBalances.filter(_.address inSetBind toDelete).delete
-
-        // Check that the balances are consistent. Still missing check that all representants are actually representants.
-        val b1 = balances.map(_.balance).sum.run.getOrElse(0)
-        val b2 = closureBalances.map(_.balance).sum.run.getOrElse(0)
-        assert(b1 == b2, s"$b1 is not equal $b2. Error updating balances!")
-
-      
-        log.info("%s balances updated in %s s, %s µs per address "
-                   format
-                   (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
-      }
+          updateAdsBalancesTable(repsAndBalances, closureBalances)
+
+          // delete all oldReps that have been unified into new ones
+          val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
+          closureBalances.filter(_.address inSetBind toDelete).delete
+
+          log.info("%s balances updated in %s s, %s µs per address "
+                     format
+                     (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
+        }
       }
     }
   }
@@ -257,7 +245,6 @@ trait BitcoinDB {
   }
 
   def insertRichestAddresses = {
-
     log.info("Calculating richest address list...")
     var startTime = System.currentTimeMillis
     DB withSession { implicit session =>
@@ -317,22 +304,6 @@ trait BitcoinDB {
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
 
-    /* block_height 
-       total_bitcoins_in_addresses XXX
-       total_transactions YYY
-       total_addresses XXX
-       total_closures XXX
-       total_addresses_with_balance YYY
-       total_closures_with_balance YYY
-       total_addresses_no_dust YYY
-       total_closures_no_dust YYY
-       gini_address 
-       gini_closure 
-       tstamp      */
-
-    // Right now with update statistics we update the values with XXX in a faster way than reading all the database. We can improve it moving the rest (YYY) to a better position, or even using psql triggers.
-    // A first approach could be to modify the values direct in ResumeBlockReader, ResumeClosure (balances can be updated whenever a utxo is added or removed)
-    // Update should only add the ginis and call saveStat
     DB withSession { implicit session =>
       val stat = currentStat
       stat.total_addresses_with_balance = balances.length.run
@@ -401,7 +372,6 @@ trait BitcoinDB {
     DB withSession { implicit session =>
 
       // MOVEMENTS
-
       // get outputs from address
       for (
         query <- List(
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index 5e3ac59..51e3296 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -32,8 +32,11 @@ class BGESpec extends FlatSpec with Matchers {
     gen(100)
     Explorer.resume
     addTxs
+    gen(10)
     Explorer.resume
-    Explorer.blockCount should be (204)
+    gen(25)
+    Explorer.resume
+    Explorer.blockCount should be (239)
     // Test config?
     //stopDocker // Deactivate it only if you want to analize the database after the tests ran
     true

From 4f93935f734d297860edb8729937849109abf14b Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Tue, 29 May 2018 00:42:40 +0200
Subject: [PATCH 43/69] Refactor updateBalances: functional. Isolated DB calls
 to make code more comprehensible

---
 src/main/scala/bge/Explorer.scala |  17 +---
 src/main/scala/db/BitcoinDB.scala | 159 +++++++++++++++++-------------
 src/test/scala/BGESpec.scala      |  23 ++---
 3 files changed, 107 insertions(+), 92 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 8e1f09c..c865ec8 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -117,6 +117,7 @@ object Explorer extends App with db.BitcoinDB {
     log.info("making new stats")
     val t: Map[Hash, Set[Hash]] = Map.empty
 
+
     for {
       key <- closure.changedReps.elements.keys
       parent = closure.changedReps.onlyFind(key) 
@@ -179,32 +180,24 @@ object Explorer extends App with db.BitcoinDB {
   }
 
   def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int)  = {
-    
+
     log.info(changedAddresses.size + " addresses changed balance")
 
     if (changedAddresses.size < balanceUpdateLimit )
     {
-      updateBalanceTables(changedAddresses.toMap, changedReps)
+      updateBalanceTables(changedAddresses.toMap, changedReps.toMap)
       insertRichestAddresses
       insertRichestClosures
       updateStatistics(changedReps,addedAds, addedReps)
     }
-    else populateStats
-        
+    else
+      populateStats
   }
 
   def populateStats = {
     createBalanceTables
-
-
-
-
-
-
     insertRichestAddresses
     insertRichestClosures
     insertStatistics
   }
-
-
 }
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 83f7d9c..584147e 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -150,84 +150,105 @@ trait BitcoinDB {
     }
   }
 
-  def updateBalanceTables(changedAddresses: scala.collection.immutable.Map[Hash, Long], changedReps: Map[Hash, Set[Hash]]) = {
-    var clock = System.currentTimeMillis
-    log.info("Updating balances ...")
-    currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
-
-    lazy val table = LmdbMap.open("closures")
-    lazy val unionFindTable = new ClosureMap(table)
-    lazy val closures = new DisjointSets[Hash](unionFindTable)
-
-    DB withSession{ implicit session => 
-      val b3 = balances.map(_.balance).sum.run.getOrElse(0)
-      val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
-
-      if (b3 != b4) {
-        log.info("Incomplete or wrong balances, generating it from scratch")
-        createBalanceTables
-      }
-      else {
-        session.withTransaction {
-          val adsAndBalances: scala.collection.immutable.Map[Hash, Long] = for ((address, change) <- changedAddresses - Hash.zero(0))
-                             yield (address,
-                                    balances.filter(_.address === Hash.hashToArray(address)).
-                                      map(_.balance).firstOption.getOrElse(0L) + change)
-
-          updateAdsBalancesTable(adsAndBalances, balances)
-
-          val repsAndChanges: collection.immutable.Map[Hash, Long] = 
-            (changedReps.map(_._1).toList ++ (changedAddresses - Hash.zero(0)).map(_._1).toList)
-              .distinct
-              .map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
-              .distinct
-              .map(r => {(r, changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum)})
-              .toMap
-
-          val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
-            (for {
-              (rep, change) <- repsAndChanges.toMap
-              representant = closures.find(rep)._1.getOrElse(rep)
-              oldies = (changedReps.getOrElse(rep, Set())+rep).map(Hash.hashToArray(_))
-            } yield (
-              representant,
-              closureBalances.filter(_.address inSetBind oldies).map(_.balance).sum.run.getOrElse(0L) + change
-            )).toMap
-
-          for ((rep, balance) <- repsAndBalances) {
-            val oldReps: Set[Hash] =
-              if (balance < 0)
-                changedReps.getOrElse(rep, Set())+rep
-              else
-                Set()
-            assert(balance >= 0, s"""
-              $rep hat negative balance $balance
-                Old value: ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
-                Old reps: ${oldReps.size} elements
-                Addresses changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size}), wallets ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
-                Changed: ${repsAndChanges.getOrElse(rep, 0)}
-                Representants/New reps: ${changedReps.map(_._2.size).sum}/${changedReps.size}
+  def assertBalancesChangedCorrectly(repsAndChanges: scala.collection.immutable.Map[Hash, Long], changedAddresses: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit =
+    DB withSession { implicit session =>
+      for ((rep, balance) <- repsAndBalances) {
+
+        val oldReps: Set[Hash] = if (balance < 0) changedReps.getOrElse(rep, Set())+rep else Set()
+
+        assert(balance >= 0, s"""
+                Start balance: ${rep.toString.take(8)} ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
+                Change balance: ${repsAndChanges.getOrElse(rep, 0)}
+                Final balance: ${rep.toString.take(8)} $balance
+                ADDRESSES ${changedAddresses.map(_._2).sum} changed ${changedAddresses.size-1}
+                ${if (changedAddresses.size < 21) (changedAddresses - Hash.zero(0)).toList.map(p => p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
+                WALLETS ${repsAndChanges.map(_._2).sum} changed ${repsAndChanges.size}
+                ${if (repsAndChanges.size < 20) repsAndChanges.toList.map(p=> p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
+                UNIONS (from/to) ${changedReps.map(_._2.size).sum}/${changedReps.size}
+                ${if (changedReps.map(_._2.size).sum < 50) changedReps.map(p => "Representant " +p._1.toString.take(8) +"\n                "+ p._2.map(_.toString.take(8)).mkString("\n                ")).mkString("\n                ") else ""}
             """)
-          }
+      }
 
-          assert(changedAddresses.map(_._2).sum == repsAndChanges.map(_._2).sum, s"""
+      assert(changedAddresses.map(_._2).sum == repsAndChanges.map(_._2).sum, s"""
                 Address changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size})
                   ${if (changedAddresses.size < 20) changedAddresses.toList.map(p => p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
                 Wallets changed ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
                   ${if (repsAndChanges.size < 20) repsAndChanges.toList.map(p=> p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
-                Representants/New reps: ${changedReps.map(_._2.size).sum} ${changedReps.size}
-          """)
+                Representants/New reps: ${changedReps.map(_._2.size).sum} ${changedReps.size} ${if (changedReps.map(_._2.size).sum < 50) changedReps.map(p => p._1.toString.take(8) + "("+ p._2.size +")").mkString("         \n") else ""}
+      """)
+  }
+
+  def checkBalances(): Boolean = DB withSession { implicit session =>
+    val b3 = balances.map(_.balance).sum.run.getOrElse(0)
+    val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
+
+    if (b3 != b4) {
+      log.info("Incomplete or wrong balances from last run")
+      false
+    } else {
+      log.info("Balances seems to be correct")
+      true
+    }
+  }
+
+  def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit =
+    DB withSession { implicit session =>
+      updateAdsBalancesTable(adsAndBalances, balances)
+      updateAdsBalancesTable(repsAndBalances, closureBalances)
+      // delete merged wallets
+      val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
+      closureBalances.filter(_.address inSetBind toDelete).delete
+    }
+
+  def getWalletBalance(a: Set[Hash]): Long = DB withSession { implicit session =>
+    closureBalances.filter(_.address inSet a.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
+  }
+
+  def getBalance(a: Hash): Long = DB withSession { implicit session =>
+    balances.filter(_.address === Hash.hashToArray(a)).map(_.balance).firstOption.getOrElse(0L)
+  }
 
-          updateAdsBalancesTable(repsAndBalances, closureBalances)
+  def updateBalanceTables(changedAddresses: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]) = {
+    val clock = System.currentTimeMillis
+    log.info("Updating balances ...")
+    currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
 
-          // delete all oldReps that have been unified into new ones
-          val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
-          closureBalances.filter(_.address inSetBind toDelete).delete
+    lazy val table = LmdbMap.open("closures")
+    lazy val unionFindTable = new ClosureMap(table)
+    lazy val closures = new DisjointSets[Hash](unionFindTable)
 
-          log.info("%s balances updated in %s s, %s µs per address "
-                     format
-                     (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
-        }
+    if (!checkBalances()) {
+      createBalanceTables
+    }
+    else {
+      DB withSession{ implicit session =>
+        val adsAndBalances: scala.collection.immutable.Map[Hash, Long] =
+          for ((address, change) <- changedAddresses - Hash.zero(0))
+             yield (address, getBalance(address) + change)
+
+        val repsAndChanges: collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
+          .distinct.map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
+          .distinct.map(r => {(r, changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum)})
+          .toMap
+
+        val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
+            (for {
+               (rep, change) <- repsAndChanges.toList
+               repe = closures.find(rep)._1.getOrElse(rep)
+               oldReps = changedReps.getOrElse(rep, Set())+rep+Hash(repe)
+               balance = getWalletBalance(oldReps)
+             } yield {
+               (repe, balance + change)
+            }).toMap
+
+        // check balances - changes
+        assertBalancesChangedCorrectly(repsAndChanges, changedAddresses, repsAndBalances, changedReps)
+          // update database
+        saveBalances(adsAndBalances, repsAndChanges, changedReps)
+
+        log.info("%s balances updated in %s s, %s µs per address "
+          format
+          (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
       }
     }
   }
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
index 51e3296..3676118 100755
--- a/src/test/scala/BGESpec.scala
+++ b/src/test/scala/BGESpec.scala
@@ -16,7 +16,7 @@ class BGESpec extends FlatSpec with Matchers {
 
   def addTxs = "/root/Bitcoin-Graph-Explorer/src/test/docker/create.sh" ! ProcessLogger(_ => ())
 
-  "Populate" should "save 10 blocks as the blockchain contains 10 blocks" in {
+  "Populate" should "work normally" in {
     startDocker 
     gen(1)
     gen(100)
@@ -24,22 +24,23 @@ class BGESpec extends FlatSpec with Matchers {
     Explorer.blockCount should be (102)
    }
 
-  "Resume" should "after add 10 blocks, resume save the 10 blocks" in {
-    gen(1)
-    gen(1)
-    Explorer.resume
-    Explorer.blockCount should be (104)
-    gen(100)
+  "Resume" should "work normally" in {
+    gen(102)
     Explorer.resume
+    Explorer.blockCount should be (204)
+
+  }
+
+  "Resume" should "work after several txs added" in {
     addTxs
     gen(10)
+    gen(100)
     Explorer.resume
-    gen(25)
+    addTxs
+    gen(1)
     Explorer.resume
-    Explorer.blockCount should be (239)
-    // Test config?
     //stopDocker // Deactivate it only if you want to analize the database after the tests ran
-    true
+    Explorer.blockCount should be (315)
   }
 }
 

From c4a7735f44fbf44d8caa8a2a59532aa3128f949d Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Tue, 29 May 2018 12:45:22 +0200
Subject: [PATCH 44/69] Fix asserts and improve code readability

---
 src/main/scala/bge/Explorer.scala             | 11 ++-
 .../scala/bge/actions/ResumeClosure.scala     |  2 -
 src/main/scala/db/BitcoinDB.scala             | 72 +++++++++----------
 3 files changed, 40 insertions(+), 45 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index c865ec8..613c878 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -154,18 +154,17 @@ object Explorer extends App with db.BitcoinDB {
   }
 
   def getWrongBlock: Option[Int] = {
-    
+
+    val lch = lastCompletedHeight
+    val bc = blockCount
     val (count,amount) = sumUTXOs
     val (countDB, amountDB) = countUTXOs
-    val bc = blockCount
     val expected = totalExpectedSatoshi(bc)
     val utxosMaxHeight = getUtxosMaxHeight 
-    
-    val lch = lastCompletedHeight
     val sameCount = count == countDB
     val sameValue = amount == amountDB
     val rightValue = amount <= expected
-    val rightBlock = (blockCount - 1 == lch) && utxosMaxHeight == lch
+    val rightBlock = (bc - 1 == lch) && utxosMaxHeight == lch
 
     if (!rightValue)  log.error("we have " + ((amount-expected)/100000000.0) + " too many bitcoins")
     if (!sameCount)   log.error("we lost utxos")
@@ -176,7 +175,7 @@ object Explorer extends App with db.BitcoinDB {
     else if (utxosMaxHeight > bc -1) Some(utxosMaxHeight)
     else if (bc - 1 > lch) Some(bc-1)
     else Some(lch)
-      //throw new Exception("This should not have happened. See logs for details.")
+
   }
 
   def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int)  = {
diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index 5b9cfcc..4dffa80 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -12,8 +12,6 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
   lazy val table = LmdbMap.open("closures")
   override lazy val unionFindTable = new ClosureMap(table)
 
-  
-
   override def insertInputsIntoTree(addressList: Iterable[Hash], tree: DisjointSets[Hash]): DisjointSets[Hash] =
   {
     val (pairList, tree1) = addressList.foldLeft((List[(Hash,Option[Hash])](),tree)){
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 584147e..31dcf27 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -169,10 +169,13 @@ trait BitcoinDB {
             """)
       }
 
-      assert(changedAddresses.map(_._2).sum == repsAndChanges.map(_._2).sum, s"""
-                Address changed ${changedAddresses.map(_._2).sum} (${changedAddresses.size})
+      val s1 = changedAddresses.filter(_._1 != Hash.zero(0)).map(_._2).sum
+      val s2 = repsAndChanges.map(_._2).sum
+
+      assert(s1 == s2, s"""
+                Address changed $s1 (${changedAddresses.size})
                   ${if (changedAddresses.size < 20) changedAddresses.toList.map(p => p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
-                Wallets changed ${repsAndChanges.map(_._2).sum} (${repsAndChanges.size})
+                Wallets changed $s2 (${repsAndChanges.size})
                   ${if (repsAndChanges.size < 20) repsAndChanges.toList.map(p=> p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
                 Representants/New reps: ${changedReps.map(_._2.size).sum} ${changedReps.size} ${if (changedReps.map(_._2.size).sum < 50) changedReps.map(p => p._1.toString.take(8) + "("+ p._2.size +")").mkString("         \n") else ""}
       """)
@@ -208,49 +211,44 @@ trait BitcoinDB {
     balances.filter(_.address === Hash.hashToArray(a)).map(_.balance).firstOption.getOrElse(0L)
   }
 
-  def updateBalanceTables(changedAddresses: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]) = {
+  def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]): Unit = {
     val clock = System.currentTimeMillis
     log.info("Updating balances ...")
     currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
-
     lazy val table = LmdbMap.open("closures")
     lazy val unionFindTable = new ClosureMap(table)
     lazy val closures = new DisjointSets[Hash](unionFindTable)
-
     if (!checkBalances()) {
       createBalanceTables
+      return
     }
-    else {
-      DB withSession{ implicit session =>
-        val adsAndBalances: scala.collection.immutable.Map[Hash, Long] =
-          for ((address, change) <- changedAddresses - Hash.zero(0))
-             yield (address, getBalance(address) + change)
-
-        val repsAndChanges: collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
-          .distinct.map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
-          .distinct.map(r => {(r, changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum)})
-          .toMap
-
-        val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
-            (for {
-               (rep, change) <- repsAndChanges.toList
-               repe = closures.find(rep)._1.getOrElse(rep)
-               oldReps = changedReps.getOrElse(rep, Set())+rep+Hash(repe)
-               balance = getWalletBalance(oldReps)
-             } yield {
-               (repe, balance + change)
-            }).toMap
-
-        // check balances - changes
-        assertBalancesChangedCorrectly(repsAndChanges, changedAddresses, repsAndBalances, changedReps)
-          // update database
-        saveBalances(adsAndBalances, repsAndChanges, changedReps)
-
-        log.info("%s balances updated in %s s, %s µs per address "
-          format
-          (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
-      }
-    }
+
+    val adsAndBalances: scala.collection.immutable.Map[Hash, Long] =
+      for ((address, change) <- changedAddresses - Hash.zero(0))
+         yield (address, getBalance(address) + change)
+
+    val repsAndChanges: collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
+        .distinct.map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
+        .distinct.map(r => {(r, changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum)})
+        .toMap
+
+    val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
+      (for {
+        (rep, change) <- repsAndChanges.toList
+        repe = closures.find(rep)._1.getOrElse(rep)
+        oldReps = changedReps.getOrElse(rep, Set())+rep+Hash(repe)
+        balance = getWalletBalance(oldReps)
+      } yield {
+        (repe, balance + change)
+      }).toMap
+
+    assertBalancesChangedCorrectly(repsAndChanges, changedAddresses, repsAndBalances, changedReps)
+    // update database
+    saveBalances(adsAndBalances, repsAndBalances, changedReps)
+
+    log.info("%s balances updated in %s s, %s µs per address "
+      format
+       (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
   }
 
   def insertRichestClosures = {

From 0bb988cdd5b4eae8c751322d9bb754f9e8060ba6 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 31 May 2018 03:30:55 +0200
Subject: [PATCH 45/69] move asserts to tests

---
 src/main/scala/bge/Explorer.scala        |  64 +++++----
 src/main/scala/bge/core/PeerSource.scala |   2 +-
 src/main/scala/db/BitcoinDB.scala        | 169 +++++++++++++----------
 src/main/scala/util/package.scala        |   9 ++
 src/test/scala/BGESpec.scala             |  46 ------
 5 files changed, 144 insertions(+), 146 deletions(-)
 delete mode 100755 src/test/scala/BGESpec.scala

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 613c878..2d52ff7 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -100,30 +100,34 @@ object Explorer extends App with db.BitcoinDB {
     insertStatistics
 
     if (!peerGroup.isRunning) startBitcoinJ
- 
+
     PopulateBlockReader
-  
+
     createIndexes
     new PopulateClosure(PopulateBlockReader.processedBlocks)
-    createAddressIndexes    
+    createAddressIndexes
     populateStats
-//    testValues
-
   }
 
   def resume = {
+    val bc = blockCount
     val read = new ResumeBlockReader
-    val closure = new ResumeClosure(read.processedBlocks)
-    log.info("making new stats")
-    val t: Map[Hash, Set[Hash]] = Map.empty
 
+      val closure = new ResumeClosure(read.processedBlocks)
+      log.info("making new stats")
+      // move me maybe when we are sure or have better tests
+      assertBalancesConsistency()
 
-    for {
-      key <- closure.changedReps.elements.keys
-      parent = closure.changedReps.onlyFind(key) 
-    } { t += (parent -> (t.getOrElse(key, Set()) + key))}
-
-    resumeStats(read.changedAddresses, t, closure.addedAds, closure.addedReps)
+      if (read.changedAddresses.size < balanceUpdateLimit) {
+        val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
+        Some(
+          (repsAndAvailable, adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap)
+        )
+      }
+      else {
+        populateStats
+        None
+      }
   }
 
   def rollBackToLastStatIfNecessary: Unit =
@@ -133,19 +137,18 @@ object Explorer extends App with db.BitcoinDB {
     }
 
   def iterateResume(newstats: Boolean = false) = {
-    // Seq("bitcoind","-daemon").run
-    
+
     if (!peerGroup.isRunning) startBitcoinJ
 
     rollBackToLastStatIfNecessary
     if (newstats) populateStats
-          
+
     while (new java.io.File(lockFile).exists)
     {
       if (blockCount > chain.getBestChainHeight)
       {
         log.info("waiting for new blocks")
-        chain.getHeightFuture(blockCount).get //wait until the chain overtakes our DB
+        waitForBitcoinJBlock(blockCount) //wait until the chain overtakes our DB
       }
       resume
     }
@@ -160,7 +163,7 @@ object Explorer extends App with db.BitcoinDB {
     val (count,amount) = sumUTXOs
     val (countDB, amountDB) = countUTXOs
     val expected = totalExpectedSatoshi(bc)
-    val utxosMaxHeight = getUtxosMaxHeight 
+    val utxosMaxHeight = getUtxosMaxHeight
     val sameCount = count == countDB
     val sameValue = amount == amountDB
     val rightValue = amount <= expected
@@ -178,19 +181,17 @@ object Explorer extends App with db.BitcoinDB {
 
   }
 
-  def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int)  = {
+  def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int):
+      (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
 
     log.info(changedAddresses.size + " addresses changed balance")
 
-    if (changedAddresses.size < balanceUpdateLimit )
-    {
-      updateBalanceTables(changedAddresses.toMap, changedReps.toMap)
-      insertRichestAddresses
-      insertRichestClosures
-      updateStatistics(changedReps,addedAds, addedReps)
-    }
-    else
-      populateStats
+    val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = updateBalanceTables(changedAddresses.toMap, changedReps.toMap)
+    insertRichestAddresses
+    insertRichestClosures
+    updateStatistics(changedReps,addedAds, addedReps)
+
+    (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances)
   }
 
   def populateStats = {
@@ -199,4 +200,9 @@ object Explorer extends App with db.BitcoinDB {
     insertRichestClosures
     insertStatistics
   }
+
+  
+  def waitForBitcoinJBlock(b: Int) = {
+    chain.getHeightFuture(b).get
+  }
 }
diff --git a/src/main/scala/bge/core/PeerSource.scala b/src/main/scala/bge/core/PeerSource.scala
index e44e214..5f346ab 100644
--- a/src/main/scala/bge/core/PeerSource.scala
+++ b/src/main/scala/bge/core/PeerSource.scala
@@ -5,7 +5,7 @@ import util._
 
 trait PeerSource extends BlockSource {
   
-  lazy val truncated = getCurrentLongestChainFromBlockCount take resumeBlockSize 
+  lazy val truncated = getCurrentLongestChainFromBlockCount take resumeBlockSize
   
   override def blockSource =
   {
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 31dcf27..89a4073 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -124,6 +124,14 @@ trait BitcoinDB {
 
        (Q.u + "create index balance_2 on closure_balances(balance)").execute
 
+      Q.updateNA("""delete from balances where balance = 0;""").execute
+      Q.updateNA("""delete from closure_balances where balance = 0;""").execute
+
+      val b3 = balances.map(_.balance).sum.run.getOrElse(0)
+      val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
+
+      assert(b3 == b4, s"""Wrong balances after fresh create!""")
+
       log.info("Balances created in %s s" format (System.currentTimeMillis - clock) / 1000)
     }
   }
@@ -150,105 +158,105 @@ trait BitcoinDB {
     }
   }
 
-  def assertBalancesChangedCorrectly(repsAndChanges: scala.collection.immutable.Map[Hash, Long], changedAddresses: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit =
-    DB withSession { implicit session =>
-      for ((rep, balance) <- repsAndBalances) {
-
-        val oldReps: Set[Hash] = if (balance < 0) changedReps.getOrElse(rep, Set())+rep else Set()
-
-        assert(balance >= 0, s"""
-                Start balance: ${rep.toString.take(8)} ${closureBalances.filter(_.address inSet oldReps.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)}
-                Change balance: ${repsAndChanges.getOrElse(rep, 0)}
-                Final balance: ${rep.toString.take(8)} $balance
-                ADDRESSES ${changedAddresses.map(_._2).sum} changed ${changedAddresses.size-1}
-                ${if (changedAddresses.size < 21) (changedAddresses - Hash.zero(0)).toList.map(p => p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
-                WALLETS ${repsAndChanges.map(_._2).sum} changed ${repsAndChanges.size}
-                ${if (repsAndChanges.size < 20) repsAndChanges.toList.map(p=> p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
-                UNIONS (from/to) ${changedReps.map(_._2.size).sum}/${changedReps.size}
-                ${if (changedReps.map(_._2.size).sum < 50) changedReps.map(p => "Representant " +p._1.toString.take(8) +"\n                "+ p._2.map(_.toString.take(8)).mkString("\n                ")).mkString("\n                ") else ""}
-            """)
-      }
-
-      val s1 = changedAddresses.filter(_._1 != Hash.zero(0)).map(_._2).sum
-      val s2 = repsAndChanges.map(_._2).sum
+  def assertBalancesConsistency(): Unit =
+    assert(getSumBalance == getSumWalletsBalance, "WTF The balances are boken again!")
 
-      assert(s1 == s2, s"""
-                Address changed $s1 (${changedAddresses.size})
-                  ${if (changedAddresses.size < 20) changedAddresses.toList.map(p => p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
-                Wallets changed $s2 (${repsAndChanges.size})
-                  ${if (repsAndChanges.size < 20) repsAndChanges.toList.map(p=> p._1.toString.take(8) + ": "+ p._2).mkString("\n                ")}
-                Representants/New reps: ${changedReps.map(_._2.size).sum} ${changedReps.size} ${if (changedReps.map(_._2.size).sum < 50) changedReps.map(p => p._1.toString.take(8) + "("+ p._2.size +")").mkString("         \n") else ""}
-      """)
+  def getSumBalance: Long = DB withSession { implicit session =>
+    balances.map(_.balance).sum.run.getOrElse(0)
   }
 
-  def checkBalances(): Boolean = DB withSession { implicit session =>
-    val b3 = balances.map(_.balance).sum.run.getOrElse(0)
-    val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
+  def getSumWalletsBalance: Long = DB withSession { implicit session =>
+    closureBalances.map(_.balance).sum.run.getOrElse(0)
+  }
 
-    if (b3 != b4) {
-      log.info("Incomplete or wrong balances from last run")
-      false
-    } else {
-      log.info("Balances seems to be correct")
-      true
-    }
+  def getUTXOSBalance(): Long = DB withSession { implicit session =>
+    utxo.map(_.value).sum.run.getOrElse(0L)
   }
 
-  def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit =
+  def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit = {
+    def a(changedReps: scala.collection.immutable.Map[Hash, Set[Hash]], repsAndBalances: scala.collection.immutable.Map[Hash, Long]) =
+      changedReps.values.fold(Set())((a, b) => a ++ b) ++ repsAndBalances.keys ++ changedReps.keys ++ adsAndBalances.keys
+
     DB withSession { implicit session =>
-      updateAdsBalancesTable(adsAndBalances, balances)
-      updateAdsBalancesTable(repsAndBalances, closureBalances)
       // delete merged wallets
-      val toDelete = changedReps.values.fold(Set())((a, b) => a ++ b).map(Hash.hashToArray(_))
+      val toDelete = a(changedReps, repsAndBalances).map(Hash.hashToArray)
       closureBalances.filter(_.address inSetBind toDelete).delete
     }
+    updateAdsBalancesTable(adsAndBalances, balances)
+    updateAdsBalancesTable(repsAndBalances, closureBalances)
+  }
 
-  def getWalletBalance(a: Set[Hash]): Long = DB withSession { implicit session =>
-    closureBalances.filter(_.address inSet a.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
+  def getWalletBalances(a: collection.immutable.Set[Hash]): Long = DB withSession { implicit session =>
+    closureBalances.filter(_.address inSetBind a.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
   }
 
   def getBalance(a: Hash): Long = DB withSession { implicit session =>
     balances.filter(_.address === Hash.hashToArray(a)).map(_.balance).firstOption.getOrElse(0L)
   }
 
-  def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]): Unit = {
+  def getAllBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
+    balances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
+  }
+
+  def getClosures(): collection.immutable.Map[Hash, Hash] = DB withSession { implicit session =>
+    addresses.sortBy(_.hash).map(p=>(p.hash, p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
+  }
+
+  def getAllWalletBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
+    closureBalances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
+  }
+
+  def getRepresentant(a: Hash): Hash = DB withSession { implicit session =>
+    addresses.filter(_.hash === Hash.hashToArray(a)).map(_.representant).run.map(Hash(_)).headOption.getOrElse(a)
+  }
+
+def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]):
+    (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long], collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
     val clock = System.currentTimeMillis
     log.info("Updating balances ...")
     currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
-    lazy val table = LmdbMap.open("closures")
-    lazy val unionFindTable = new ClosureMap(table)
-    lazy val closures = new DisjointSets[Hash](unionFindTable)
-    if (!checkBalances()) {
-      createBalanceTables
-      return
-    }
 
     val adsAndBalances: scala.collection.immutable.Map[Hash, Long] =
       for ((address, change) <- changedAddresses - Hash.zero(0))
          yield (address, getBalance(address) + change)
 
+    // FIXME refactor
     val repsAndChanges: collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
-        .distinct.map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
-        .distinct.map(r => {(r, changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum)})
-        .toMap
-
-    val repsAndBalances: scala.collection.immutable.Map[Hash, Long] =
-      (for {
-        (rep, change) <- repsAndChanges.toList
-        repe = closures.find(rep)._1.getOrElse(rep)
-        oldReps = changedReps.getOrElse(rep, Set())+rep+Hash(repe)
-        balance = getWalletBalance(oldReps)
-      } yield {
-        (repe, balance + change)
-      }).toMap
-
-    assertBalancesChangedCorrectly(repsAndChanges, changedAddresses, repsAndBalances, changedReps)
+      .distinct
+      .map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
+      .distinct
+      .map(r => (r,
+             changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum))
+      .groupBy(p=>getRepresentant(p._1))
+      .map(p=> (p._1, p._2.map(_._2).sum))
+      .toSeq
+      .sortBy(_._1)
+      .toMap
+
+    val e: collection.immutable.Set[Hash] = collection.immutable.Set()
+    val repsAndAvailable: scala.collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
+      .distinct
+      .map(p => (p,
+             if (changedReps contains p) (changedReps(p)+p) else collection.immutable.Set(getRepresentant(p))))
+      .groupBy(p=>getRepresentant(p._1))
+      .map(p=> (p._1,
+             getWalletBalances(p._2.foldLeft(e)((s,t) => s++t._2) + p._1)))
+      .toSeq
+      .sortBy(_._1)
+      .toMap
+
+
+    val repsAndBalances: collection.immutable.Map[Hash, Long] = (repsAndAvailable zip repsAndChanges)
+      .map(p=> if (p._1._1 != p._2._1) throw new Error(s"WTF ${getRepresentant(p._1._1)} ${getRepresentant(p._2._1)}") else (p._1._1, p._1._2 + p._2._2))
+
     // update database
     saveBalances(adsAndBalances, repsAndBalances, changedReps)
 
     log.info("%s balances updated in %s s, %s µs per address "
       format
-       (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
+               (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
+
+       (repsAndAvailable, adsAndBalances,repsAndChanges, repsAndBalances)
   }
 
   def insertRichestClosures = {
@@ -294,7 +302,10 @@ trait BitcoinDB {
 
     val startTime = System.currentTimeMillis
     DB withSession { implicit session =>
+
+      
       val query = """
+       delete from stats where block_height = (select coalesce(max(block_height),0) from blocks);
        insert
        into stats select
        (select coalesce(max(block_height),0) from blocks),
@@ -319,10 +330,13 @@ trait BitcoinDB {
   def updateStatistics(changedReps: Map[Hash, Set[Hash]], addedAds: Int, addedReps: Int) = {
 
     log.info("Updating stats")
+
     val time = System.currentTimeMillis
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
-
+    var a = changedReps.values.map(_.size).sum
+    // FIXME: addReps and changedRes are wrong!
+    // assert(addedReps - a >= 0, s"we lost the count of wallets $addedReps > $a => $changedReps")
     DB withSession { implicit session =>
       val stat = currentStat
       stat.total_addresses_with_balance = balances.length.run
@@ -422,10 +436,25 @@ trait BitcoinDB {
   }
 
   def saveStat(stat: Stat) = DB withSession { implicit session =>
+    stats.filter(_.block_height === stat.block_height).delete
     stats.insert(Stat.unapply(stat).get)
   }
 
-  case class Stat(var block_height: Int, var total_bitcoins_in_addresses: Long, var total_transactions: Long, var total_addresses: Long, var total_closures: Long, var total_addresses_with_balance: Long, var total_closures_with_balance: Long, var total_addresses_no_dust: Long, var total_closures_no_dust: Long, var gini_closure: Double, var gini_address: Double, var tstamp: Long)
+  case class Stat(var block_height: Int, var total_bitcoins_in_addresses: Long, var total_transactions: Long, var total_addresses: Long, var total_closures: Long, var total_addresses_with_balance: Long, var total_closures_with_balance: Long, var total_addresses_no_dust: Long, var total_closures_no_dust: Long, var gini_closure: Double, var gini_address: Double, var tstamp: Long){
+    override def equals(that: Any): Boolean = {
+      def getCCParams(cc: AnyRef) =
+        (Map[String, Any]() /: cc.getClass.getDeclaredFields) {(a, f) =>
+          f.setAccessible(true)
+          a + (f.getName -> f.get(cc))
+        }
+      that match {
+        case that: Stat => 
+          getCCParams(this)-"tstamp" == getCCParams(that)-"tstamp"
+        case _ =>
+          false
+      }
+    }
+  }
 
   def lastCompletedHeight: Int = DB withSession { implicit session =>
     stats.map(_.block_height).max.run.getOrElse(0)
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index edda35a..b2fbc4a 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -99,6 +99,15 @@ package object util
     }
   }
 
+
   
+  def convertToMap(a: DisjointSets[Hash]): collection.mutable.Map[Hash, Set[Hash]] = {
+    val t: collection.mutable.Map[Hash, Set[Hash]] = collection.mutable.Map.empty
+    for {
+      key <- a.elements.keys
+      parent = a.onlyFind(key)
+    } { t += (parent -> (t.getOrElse(key, Set()) + key))}
+    t
+  }
 }
 
diff --git a/src/test/scala/BGESpec.scala b/src/test/scala/BGESpec.scala
deleted file mode 100755
index 3676118..0000000
--- a/src/test/scala/BGESpec.scala
+++ /dev/null
@@ -1,46 +0,0 @@
-import sys.process._
-import util._
-import org.scalatest._
-
-class BGESpec extends FlatSpec with Matchers {
-
-  // FIXME: Use bitcoin rpc client and docker tools instead 
-
-  val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
-
-  def gen(i:Int) = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
-
-  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! ProcessLogger(_ => ())
-
-  def stopDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/stop.sh" ! ProcessLogger(_ => ())
-
-  def addTxs = "/root/Bitcoin-Graph-Explorer/src/test/docker/create.sh" ! ProcessLogger(_ => ())
-
-  "Populate" should "work normally" in {
-    startDocker 
-    gen(1)
-    gen(100)
-    Explorer.populate
-    Explorer.blockCount should be (102)
-   }
-
-  "Resume" should "work normally" in {
-    gen(102)
-    Explorer.resume
-    Explorer.blockCount should be (204)
-
-  }
-
-  "Resume" should "work after several txs added" in {
-    addTxs
-    gen(10)
-    gen(100)
-    Explorer.resume
-    addTxs
-    gen(1)
-    Explorer.resume
-    //stopDocker // Deactivate it only if you want to analize the database after the tests ran
-    Explorer.blockCount should be (315)
-  }
-}
-

From 7d1c67c71622cd819510f1694bbd7a9ffd869eef Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 31 May 2018 16:32:43 +0200
Subject: [PATCH 46/69] changes

---
 src/main/scala/db/BitcoinDB.scala             |  1 +
 src/test/scala/ExplorerIntegrationTests.scala | 69 ++++++++++++++
 src/test/scala/tools/Tool.scala               | 92 +++++++++++++++++++
 3 files changed, 162 insertions(+)
 create mode 100755 src/test/scala/ExplorerIntegrationTests.scala
 create mode 100644 src/test/scala/tools/Tool.scala

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 89a4073..729efa9 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -246,6 +246,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
       .toMap
 
 
+  // FIXME it should never happen
     val repsAndBalances: collection.immutable.Map[Hash, Long] = (repsAndAvailable zip repsAndChanges)
       .map(p=> if (p._1._1 != p._2._1) throw new Error(s"WTF ${getRepresentant(p._1._1)} ${getRepresentant(p._2._1)}") else (p._1._1, p._1._2 + p._2._2))
 
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
new file mode 100755
index 0000000..bf655d0
--- /dev/null
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -0,0 +1,69 @@
+import sys.process._
+import org.scalatest._
+import util.Hash
+import tools.Tool
+
+class ExplorerIntegrationTests extends FlatSpec with Matchers {
+
+  "docker" should "start correctly" in {
+    startDocker should be (0)
+  }
+
+  it should "communicate bitcoin data" in {
+    gen(100) should be (0)
+    gen(1) should be (0)
+    gen(1) should be (0)
+    addTx(6) should be (0)
+    gen(1) should be (0)
+  }
+
+  "explorer" should "throw postgres exception if there are no blocks" in {
+    a [org.postgresql.util.PSQLException] should be thrownBy {
+      blockCount should be (0)
+    }
+  }
+
+  it should "populate blocks" in {
+    savePopulate
+    blockCount should be (count+1)
+  }
+
+  it should "have same stats using create or update" in {
+    val initialStat = currentStat
+    gen(1) should be (0)
+    saveResume
+    currentStat.total_closures >= initialStat.total_closures should be (true)
+    blockCount should be (count+1)
+    gen(1) should be (0)
+  }
+
+  it should "add correclty blocks" in {
+    saveResume
+    blockCount should be (count+1)
+  }
+
+  it should "add blocks and txs" in {
+    addTx(6) should be (0)
+    gen(1) should be (0)
+    gen(1) should be (0)
+    addTx(7) should be (0)
+    gen(1) should be (0)
+    saveResume
+    blockCount should be (count+1)
+  }
+
+  for (i <- 1 to 10) {
+    it should "work after several rollbacks"+i in {
+      val initialStat = currentStat
+      rollBack(blockCount-1)
+      saveResume
+      blockCount should be (count+1)
+      currentStat should be (initialStat)
+      addTx(8)
+      addTx(8)
+      gen(2) should be (0)
+      saveResume
+      blockCount should be (count+1)
+    }
+  }
+}
diff --git a/src/test/scala/tools/Tool.scala b/src/test/scala/tools/Tool.scala
new file mode 100644
index 0000000..b9abe1b
--- /dev/null
+++ b/src/test/scala/tools/Tool.scala
@@ -0,0 +1,92 @@
+package tools
+
+import Explorer._
+import util.Hash
+
+object Tools {
+
+  val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
+
+  def gen(i:Int): Int = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
+
+  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! ProcessLogger(_ => ())
+
+  def stopDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/stop.sh" ! ProcessLogger(_ => ())
+
+  def addTx(n: Long) = ("/root/Bitcoin-Graph-Explorer/src/test/docker/create.sh " + n.toString) ! ProcessLogger(_ => ())
+
+  def count: Int = ((BITCOIN + " getblockcount") !!).trim.toInt
+
+  def saveResume = {
+    val init = currentStat
+    resume match {
+      case Some((a,b,c,d,e,f)) =>
+        assertBalancesUpdatedCorrectly(a,b,c,d,e,f)
+      case None =>
+        assertBalancesCreatedCorrectly
+    }
+
+    currentStat.total_closures >= init.total_closures should be (true)
+    currentStat.total_addresses >= init.total_addresses should be (true)
+  }
+
+  def savePopulate = {
+    populate
+    assertBalancesCreatedCorrectly
+  }
+
+  def s: collection.immutable.Map[Hash, Long] = collection.immutable.Map()
+
+  def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
+
+  def assertBalancesCreatedCorrectly =
+    assertBalancesUpdatedCorrectly(s,s,s,s,s,x)
+
+  def assertBalancesUpdatedCorrectly(
+    repsAndAvailable: collection.immutable.Map[Hash, Long],
+    adsAndBalances: collection.immutable.Map[Hash, Long],
+    repsAndChanges: collection.immutable.Map[Hash, Long],
+    changedAddresses: collection.immutable.Map[Hash, Long],
+    repsAndBalances: collection.immutable.Map[Hash, Long],
+    changedReps: collection.immutable.Map[Hash, Set[Hash]] ): Unit = {
+    lazy val ca = changedAddresses - Hash.zero(0)
+    lazy val s1 = ca.map(_._2).sum
+    lazy val s2 = repsAndChanges.map(_._2).sum
+    lazy val b3 = getSumBalance
+    lazy val b4 = getSumWalletsBalance
+    lazy val message = s"""
+  WALLETSs and CHANGES ${str1(repsAndChanges)}
+  ADDRESSs and CHANGES ${str1(ca)}
+  ADDED REPS ${str1(repsAndBalances)}
+  REPS TABLE  ${str2(getClosures())}
+  ADDs BALANCES ${str1(getAllBalances())}
+  REPS BALANCES ${str1(getAllWalletBalances())}
+  WALLETS CHANGED MORE THAN ADDRESSES ###
+    ${nr(s2-s1)} btcs
+  WALLETs TABLES MORE THAN ADDRESSES ###
+    ${nr(b4-b3)} btcs
+  UTXO / ADDRESS / WALLET
+    ${nr(getUTXOSBalance())} / ${nr(b3)} / ${nr(b4)}
+  ${changedReps.map(p=> p._1.toString.drop(2).take(4)+" -> " + p._2.map(_.toString.drop(2).take(4)))}
+"""
+
+    for ((rep, balance) <- repsAndBalances)
+      if (balance < 0)
+        throw new Error(message + "  NEGATIVE VALUE!!!")
+      else if (rep != getRepresentant(rep))
+        throw new Error(message + "  WTH")
+
+    if (s1 != s2)
+      throw new Error(message + "WALLETs CHANGED MORE THAN ADDRESSES!!!")
+    else if (b3 != b4)
+      throw new Error(message  +"  WALLETs TABLE HAS MORE THAN ADDRESSES!!!")
+  }
+
+  // Some help formatter
+  def nr(b: Long): String = ((b/10000)/10000.0).toString
+
+  def str1(n: collection.immutable.Map[Hash, Long]): String = if (n.size> 0 && n.size < 25) "#: " + n.size + "  total: " + nr(n.map(_._2).sum) +"\n    " + n.toSeq.sortBy(_._1.toString.drop(2).take(4)).map(m=>m._1.toString.drop(2).take(4)+": "+(nr(m._2))).mkString("\n    ") else n.size.toString
+
+  def str2(n: collection.immutable.Map[Hash, Hash]): String = if (n.size > 0 && n.size < 25) "(" + n.size + ")\n    " + n.toSeq.sortBy(_._1.toString.drop(2).take(4)).map(m=>m._1.toString.drop(2).take(4)+" => "+(m._2.toString.drop(2).take(4))).mkString("\n    ") else n.size.toString
+
+}

From 8c954c364ce1aa4ae565392be46fc2254c21acb0 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Thu, 31 May 2018 18:31:17 +0200
Subject: [PATCH 47/69] move explorer logic to testExplorer tools class. add
 missing bitcoin scripts

---
 src/main/scala/bge/Explorer.scala             |  2 +-
 src/main/scala/db/BitcoinDB.scala             |  5 +-
 src/test/docker/create.sh                     |  5 +
 src/test/scala/ExplorerIntegrationTests.scala | 49 +++++-----
 .../{tools/Tool.scala => TestExplorer.scala}  | 92 +++++++++++--------
 5 files changed, 86 insertions(+), 67 deletions(-)
 create mode 100755 src/test/docker/create.sh
 rename src/test/scala/{tools/Tool.scala => TestExplorer.scala} (52%)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 2d52ff7..891f3ee 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -121,7 +121,7 @@ object Explorer extends App with db.BitcoinDB {
       if (read.changedAddresses.size < balanceUpdateLimit) {
         val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
         Some(
-          (repsAndAvailable, adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap)
+          (repsAndAvailable, adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap, closure.addedAds, closure.addedReps)
         )
       }
       else {
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 729efa9..6892e7d 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -304,7 +304,6 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     val startTime = System.currentTimeMillis
     DB withSession { implicit session =>
 
-      
       val query = """
        delete from stats where block_height = (select coalesce(max(block_height),0) from blocks);
        insert
@@ -335,9 +334,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     val time = System.currentTimeMillis
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
-    var a = changedReps.values.map(_.size).sum
-    // FIXME: addReps and changedRes are wrong!
-    // assert(addedReps - a >= 0, s"we lost the count of wallets $addedReps > $a => $changedReps")
+
     DB withSession { implicit session =>
       val stat = currentStat
       stat.total_addresses_with_balance = balances.length.run
diff --git a/src/test/docker/create.sh b/src/test/docker/create.sh
new file mode 100755
index 0000000..66ebc68
--- /dev/null
+++ b/src/test/docker/create.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+CMD="docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333 "
+ADD1=$(${CMD} getnewaddress)
+X=$(${CMD} sendtoaddress ${ADD1} $1)
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index bf655d0..317246b 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -1,7 +1,6 @@
-import sys.process._
+
+import TestExplorer._
 import org.scalatest._
-import util.Hash
-import tools.Tool
 
 class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
@@ -19,27 +18,25 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
   "explorer" should "throw postgres exception if there are no blocks" in {
     a [org.postgresql.util.PSQLException] should be thrownBy {
-      blockCount should be (0)
+      bgeCount should be (0)
     }
   }
 
   it should "populate blocks" in {
     savePopulate
-    blockCount should be (count+1)
+    bgeCount should be (bitcoinCount)
   }
 
   it should "have same stats using create or update" in {
-    val initialStat = currentStat
     gen(1) should be (0)
-    saveResume
-    currentStat.total_closures >= initialStat.total_closures should be (true)
-    blockCount should be (count+1)
+    saveResume should be (None)
+    bgeCount should be (bitcoinCount)
     gen(1) should be (0)
   }
 
   it should "add correclty blocks" in {
-    saveResume
-    blockCount should be (count+1)
+    saveResume should be (None)
+    bgeCount should be (bitcoinCount)
   }
 
   it should "add blocks and txs" in {
@@ -48,22 +45,22 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     gen(1) should be (0)
     addTx(7) should be (0)
     gen(1) should be (0)
-    saveResume
-    blockCount should be (count+1)
+    addTx(7) should be (0)
+    gen(1) should be (0)
+    saveResume should be (None)
+    bgeCount should be (bitcoinCount)
   }
 
-  for (i <- 1 to 10) {
-    it should "work after several rollbacks"+i in {
-      val initialStat = currentStat
-      rollBack(blockCount-1)
-      saveResume
-      blockCount should be (count+1)
-      currentStat should be (initialStat)
-      addTx(8)
-      addTx(8)
-      gen(2) should be (0)
-      saveResume
-      blockCount should be (count+1)
-    }
+  it should "work after several rollbacks" in {
+    val initialStat = stat
+    rollBackLast
+    saveResume should be (None)
+    bgeCount should be (bitcoinCount)
+    stat should be (initialStat)
+    addTx(8)
+    addTx(8)
+    gen(2) should be (0)
+    saveResume should be (None)
+    bgeCount should be (bitcoinCount)
   }
 }
diff --git a/src/test/scala/tools/Tool.scala b/src/test/scala/TestExplorer.scala
similarity index 52%
rename from src/test/scala/tools/Tool.scala
rename to src/test/scala/TestExplorer.scala
index b9abe1b..090b7f8 100644
--- a/src/test/scala/tools/Tool.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -1,9 +1,11 @@
-package tools
-
-import Explorer._
 import util.Hash
+import Explorer._
+import sys.process._
+
 
-object Tools {
+object TestExplorer {
+
+  // BITCOIN-CLI functions
 
   val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
 
@@ -15,19 +17,24 @@ object Tools {
 
   def addTx(n: Long) = ("/root/Bitcoin-Graph-Explorer/src/test/docker/create.sh " + n.toString) ! ProcessLogger(_ => ())
 
-  def count: Int = ((BITCOIN + " getblockcount") !!).trim.toInt
+  def bitcoinCount: Int = ((BITCOIN + " getblockcount") !!).trim.toInt+1
+
+  // EXPLORER functions
+
+  def bgeCount = blockCount
+
+  def stat = currentStat
+
+  def rollBackLast = rollBack(blockCount-1)
 
   def saveResume = {
     val init = currentStat
     resume match {
-      case Some((a,b,c,d,e,f)) =>
-        assertBalancesUpdatedCorrectly(a,b,c,d,e,f)
+      case Some((a,b,c,d,e,f,x,y)) =>
+        assertBalancesUpdatedCorrectly(a,b,c,d,e,f,x,y)
       case None =>
         assertBalancesCreatedCorrectly
     }
-
-    currentStat.total_closures >= init.total_closures should be (true)
-    currentStat.total_addresses >= init.total_addresses should be (true)
   }
 
   def savePopulate = {
@@ -40,7 +47,7 @@ object Tools {
   def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
 
   def assertBalancesCreatedCorrectly =
-    assertBalancesUpdatedCorrectly(s,s,s,s,s,x)
+    assertBalancesUpdatedCorrectly(s,s,s,s,s,x,0,0)
 
   def assertBalancesUpdatedCorrectly(
     repsAndAvailable: collection.immutable.Map[Hash, Long],
@@ -48,38 +55,51 @@ object Tools {
     repsAndChanges: collection.immutable.Map[Hash, Long],
     changedAddresses: collection.immutable.Map[Hash, Long],
     repsAndBalances: collection.immutable.Map[Hash, Long],
-    changedReps: collection.immutable.Map[Hash, Set[Hash]] ): Unit = {
+    changedReps: collection.immutable.Map[Hash, Set[Hash]],
+    addedAdds: Int,
+    addedReps: Int): Option[String] = {
+
     lazy val ca = changedAddresses - Hash.zero(0)
     lazy val s1 = ca.map(_._2).sum
     lazy val s2 = repsAndChanges.map(_._2).sum
     lazy val b3 = getSumBalance
     lazy val b4 = getSumWalletsBalance
-    lazy val message = s"""
-  WALLETSs and CHANGES ${str1(repsAndChanges)}
-  ADDRESSs and CHANGES ${str1(ca)}
-  ADDED REPS ${str1(repsAndBalances)}
-  REPS TABLE  ${str2(getClosures())}
-  ADDs BALANCES ${str1(getAllBalances())}
-  REPS BALANCES ${str1(getAllWalletBalances())}
-  WALLETS CHANGED MORE THAN ADDRESSES ###
-    ${nr(s2-s1)} btcs
-  WALLETs TABLES MORE THAN ADDRESSES ###
-    ${nr(b4-b3)} btcs
-  UTXO / ADDRESS / WALLET
-    ${nr(getUTXOSBalance())} / ${nr(b3)} / ${nr(b4)}
-  ${changedReps.map(p=> p._1.toString.drop(2).take(4)+" -> " + p._2.map(_.toString.drop(2).take(4)))}
+    lazy val message2 = s"""
+      WALLETSs and CHANGES ${str1(repsAndChanges)}
+      ADDRESSs and CHANGES ${str1(ca)}
+      ADDED REPS ${str1(repsAndBalances)}
+      REPS TABLE  ${str2(getClosures())}
+      ADDs BALANCES ${str1(getAllBalances())}
+      REPS BALANCES ${str1(getAllWalletBalances())}
+      WALLETS CHANGED MORE THAN ADDRESSES ###
+        ${nr(s2-s1)} btcs
+      WALLETs TABLES MORE THAN ADDRESSES ###
+        ${nr(b4-b3)} btcs
+      UTXO / ADDRESS / WALLET
+        ${nr(getUTXOSBalance())} / ${nr(b3)} / ${nr(b4)}
+      ${changedReps.map(p=> p._1.toString.drop(2).take(4)+" -> " + p._2.map(_.toString.drop(2).take(4)))}
 """
-
-    for ((rep, balance) <- repsAndBalances)
-      if (balance < 0)
-        throw new Error(message + "  NEGATIVE VALUE!!!")
-      else if (rep != getRepresentant(rep))
-        throw new Error(message + "  WTH")
-
-    if (s1 != s2)
-      throw new Error(message + "WALLETs CHANGED MORE THAN ADDRESSES!!!")
+    lazy val message = ""
+
+    // test updateBalances
+    if (adsAndBalances.exists(r => r._2 < 0))
+      Some(message + "  NEGATIVE ADDRESS - updateBalances")
+    else if(repsAndBalances.exists(r => r._2 < 0))
+      Some(message + "  NEGATIVE WALLET - updateBalances")
+    else if (repsAndBalances.exists(r => r._1 != getRepresentant(r._1)))
+      Some(message + "  WRONG UPDATE - updateBalances")
+    else if (s1 != s2)
+      Some(message + "  WRONG CHANGED VALUES - updateBalances")
     else if (b3 != b4)
-      throw new Error(message  +"  WALLETs TABLE HAS MORE THAN ADDRESSES!!!")
+      Some(message  +"  WRONG CHANGED BALANCE - updateBalances")
+    // test updateStatistics
+    else if (addedReps <= changedReps.values.map(_.size).sum)
+      Some(message + "  LOST WALLETS - updateStatistics")
+    else if (addedAdds < 0)
+      Some(message + "  LOST ADDRESSES - updateStatistics")
+    else
+      None
+
   }
 
   // Some help formatter

From 1f0ce73d4d60dd6817eb3480b12ac4cf6409d04b Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 1 Jun 2018 01:21:09 +0200
Subject: [PATCH 48/69] update balances errors are catched by test suite

---
 src/main/scala/bge/Explorer.scala             | 57 ++++++------
 .../scala/bge/actions/ResumeClosure.scala     |  1 +
 src/main/scala/bge/core/FastBlockReader.scala |  2 +-
 src/main/scala/bge/core/PeerSource.scala      |  4 +-
 src/main/scala/db/BitcoinDB.scala             | 93 +++++++++++--------
 src/test/scala/ExplorerIntegrationTests.scala | 92 +++++++++++-------
 src/test/scala/TestExplorer.scala             | 85 +++++++++--------
 7 files changed, 193 insertions(+), 141 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 891f3ee..63d42b9 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -113,40 +113,45 @@ object Explorer extends App with db.BitcoinDB {
     val bc = blockCount
     val read = new ResumeBlockReader
 
-      val closure = new ResumeClosure(read.processedBlocks)
-      log.info("making new stats")
-      // move me maybe when we are sure or have better tests
-      assertBalancesConsistency()
-
-      if (read.changedAddresses.size < balanceUpdateLimit) {
-        val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
-        Some(
-          (repsAndAvailable, adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap, closure.addedAds, closure.addedReps)
-        )
-      }
-      else {
-        populateStats
-        None
-      }
+    val closure = new ResumeClosure(read.processedBlocks)
+    // move me maybe when we are sure or have better tests
+    val checkBalances =  getSumBalance == getSumWalletsBalance
+
+    if (!checkBalances)
+      log.info("Error updating balances .... create it again")
+
+    if (read.changedAddresses.size < balanceUpdateLimit && checkBalances) {
+      val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
+      Some(
+        (repsAndAvailable, adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap, closure.addedAds, closure.addedReps)
+      )
+    }
+    else {
+      populateStats
+      None
+    }
   }
 
-  def rollBackToLastStatIfNecessary: Unit =
-    for (block <- getWrongBlock){
-      rollBack(block)
-      rollBackToLastStatIfNecessary
+  def rollBackToLastStatIfNecessary: Boolean =
+    getWrongBlock match {
+      case None =>
+        false
+      case Some(block: Int) => 
+        rollBack(block)
+        rollBackToLastStatIfNecessary
+        true
     }
 
   def iterateResume(newstats: Boolean = false) = {
 
-    if (!peerGroup.isRunning) startBitcoinJ
+    if (!peerGroup.isRunning)
+      startBitcoinJ
 
-    rollBackToLastStatIfNecessary
-    if (newstats) populateStats
+    if (rollBackToLastStatIfNecessary || newstats)
+      populateStats
 
-    while (new java.io.File(lockFile).exists)
-    {
-      if (blockCount > chain.getBestChainHeight)
-      {
+    while (new java.io.File(lockFile).exists) {
+      if (blockCount > chain.getBestChainHeight) {
         log.info("waiting for new blocks")
         waitForBitcoinJBlock(blockCount) //wait until the chain overtakes our DB
       }
diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index 4dffa80..aa34984 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -51,6 +51,7 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
             val updateQuery = for(p <- addresses if p.representant === hashToArray(oldRep)) yield p.representant
             updateQuery.update(newRep) // TODO: compile query
             recordChangedRep(newRep, oldRep)
+ 
           case _ => addRepFlag = 0 // one of the elements had this rep before => don't count a new one
         }
     }
diff --git a/src/main/scala/bge/core/FastBlockReader.scala b/src/main/scala/bge/core/FastBlockReader.scala
index ff565f3..48119d3 100644
--- a/src/main/scala/bge/core/FastBlockReader.scala
+++ b/src/main/scala/bge/core/FastBlockReader.scala
@@ -73,7 +73,7 @@ trait FastBlockReader extends BlockReader {
   def finishBlock(b: Hash, txs: Int, btcs: Long, tstamp: Long, height:Int) = {
     processedBlocks :+= height
     insertBlock(b, height, txs, btcs, tstamp)
-    log.info("Saved block " + height + " consisting of " + txs + " txs")
+    //log.info("Saved block " + height + " consisting of " + txs + " txs")
   }
 
   def pre  = {
diff --git a/src/main/scala/bge/core/PeerSource.scala b/src/main/scala/bge/core/PeerSource.scala
index 5f346ab..a500de1 100644
--- a/src/main/scala/bge/core/PeerSource.scala
+++ b/src/main/scala/bge/core/PeerSource.scala
@@ -15,9 +15,9 @@ trait PeerSource extends BlockSource {
 
     for ((blockHash,no) <- truncated.toIterator) yield {
       val future = peer.getBlock(blockHash)
-      log.info("Waiting for node to send us the requested block: " + blockHash)
+//      log.info("Waiting for node to send us the requested block: " + blockHash)
       val block = future.get()
-      log.info("Block received")
+//      log.info("Block received")
       (block,no)
     }
 
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 6892e7d..30dd3dd 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -158,9 +158,6 @@ trait BitcoinDB {
     }
   }
 
-  def assertBalancesConsistency(): Unit =
-    assert(getSumBalance == getSumWalletsBalance, "WTF The balances are boken again!")
-
   def getSumBalance: Long = DB withSession { implicit session =>
     balances.map(_.balance).sum.run.getOrElse(0)
   }
@@ -174,42 +171,23 @@ trait BitcoinDB {
   }
 
   def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit = {
-    def a(changedReps: scala.collection.immutable.Map[Hash, Set[Hash]], repsAndBalances: scala.collection.immutable.Map[Hash, Long]) =
-      changedReps.values.fold(Set())((a, b) => a ++ b) ++ repsAndBalances.keys ++ changedReps.keys ++ adsAndBalances.keys
 
     DB withSession { implicit session =>
       // delete merged wallets
-      val toDelete = a(changedReps, repsAndBalances).map(Hash.hashToArray)
+      val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ repsAndBalances.keys ++ changedReps.keys).map(Hash.hashToArray)
       closureBalances.filter(_.address inSetBind toDelete).delete
     }
     updateAdsBalancesTable(adsAndBalances, balances)
     updateAdsBalancesTable(repsAndBalances, closureBalances)
   }
 
-  def getWalletBalances(a: collection.immutable.Set[Hash]): Long = DB withSession { implicit session =>
-    closureBalances.filter(_.address inSetBind a.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
-  }
-
-  def getBalance(a: Hash): Long = DB withSession { implicit session =>
-    balances.filter(_.address === Hash.hashToArray(a)).map(_.balance).firstOption.getOrElse(0L)
-  }
-
-  def getAllBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
-    balances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
-  }
-
-  def getClosures(): collection.immutable.Map[Hash, Hash] = DB withSession { implicit session =>
-    addresses.sortBy(_.hash).map(p=>(p.hash, p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
-  }
-
-  def getAllWalletBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
-    closureBalances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
-  }
-
+  // Unsave functions, use only for testing
+  // end of unsave functions block
   def getRepresentant(a: Hash): Hash = DB withSession { implicit session =>
     addresses.filter(_.hash === Hash.hashToArray(a)).map(_.representant).run.map(Hash(_)).headOption.getOrElse(a)
   }
 
+
 def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]):
     (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long], collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
     val clock = System.currentTimeMillis
@@ -233,19 +211,17 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
       .sortBy(_._1)
       .toMap
 
-    val e: collection.immutable.Set[Hash] = collection.immutable.Set()
     val repsAndAvailable: scala.collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
       .distinct
       .map(p => (p,
              if (changedReps contains p) (changedReps(p)+p) else collection.immutable.Set(getRepresentant(p))))
       .groupBy(p=>getRepresentant(p._1))
       .map(p=> (p._1,
-             getWalletBalances(p._2.foldLeft(e)((s,t) => s++t._2) + p._1)))
+             getWalletBalances(p._2.foldLeft(Set(Hash.zero(0)))((s,t) => s++t._2) + p._1)))
       .toSeq
       .sortBy(_._1)
       .toMap
 
-
   // FIXME it should never happen
     val repsAndBalances: collection.immutable.Map[Hash, Long] = (repsAndAvailable zip repsAndChanges)
       .map(p=> if (p._1._1 != p._2._1) throw new Error(s"WTF ${getRepresentant(p._1._1)} ${getRepresentant(p._2._1)}") else (p._1._1, p._1._2 + p._2._2))
@@ -294,6 +270,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     }
   }
 
+
   def insertStatistics = {
 
     val (nonDustAddresses, addressGini) = getGini(balances)
@@ -334,6 +311,9 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     val time = System.currentTimeMillis
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
+    val addedClosures = addedReps
+      - (changedReps.values.foldLeft(Set[Hash]())((s,p) => s++p)--changedReps.keys).size
+      + changedReps.keys.size
 
     DB withSession { implicit session =>
       val stat = currentStat
@@ -348,7 +328,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
       stat.total_transactions = blockDB.map(_.txs).filter(_ > 0).sum.run.getOrElse(0).toLong
       stat.total_bitcoins_in_addresses = balances.map(_.balance).sum.run.getOrElse(0L) / 100000000
       stat.total_addresses += addedAds
-      stat.total_closures += addedReps - changedReps.values.map(_.size).sum
+      stat.total_closures += addedClosures
       saveStat(stat)
       log.info("Updated in " + (System.currentTimeMillis - time) / 1000 + " seconds")
     }
@@ -402,14 +382,10 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
 
     DB withSession { implicit session =>
 
-      // MOVEMENTS
-      // get outputs from address
       for (
         query <- List(
           "create index address on movements (address);",
           """create unique index tx_idx  on movements (transaction_hash, "index");""",
-          //  "create index  spent_in_transaction_hash2 on movements (spent_in_transaction_hash, address);",
-          // not needed for closure
           "create index spent_in on movements(spent_in_transaction_hash)", // for api/Movements
           "create index height_in on movements (height_in);", // for closure
           "create index height_out_in on movements (height_out, height_in);",
@@ -422,8 +398,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
         Q.updateNA(query).execute
         log.info("Finished" + query)
       }
-
-    }
+   }
 
     log.info("Indexes created in %s s" format (System.currentTimeMillis - time) / 1000)
 
@@ -438,8 +413,18 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     stats.insert(Stat.unapply(stat).get)
   }
 
-  case class Stat(var block_height: Int, var total_bitcoins_in_addresses: Long, var total_transactions: Long, var total_addresses: Long, var total_closures: Long, var total_addresses_with_balance: Long, var total_closures_with_balance: Long, var total_addresses_no_dust: Long, var total_closures_no_dust: Long, var gini_closure: Double, var gini_address: Double, var tstamp: Long){
-    override def equals(that: Any): Boolean = {
+  case class Stat(var block_height: Int,
+                  var total_bitcoins_in_addresses: Long,
+                  var total_transactions: Long,
+                  var total_addresses: Long,
+                  var total_closures: Long,
+                  var total_addresses_with_balance: Long,
+                  var total_closures_with_balance: Long,
+                  var total_addresses_no_dust: Long,
+                  var total_closures_no_dust: Long,
+                  var gini_closure: Double,
+                  var gini_address: Double,
+                  var tstamp: Long) { override def equals(that: Any): Boolean = {
       def getCCParams(cc: AnyRef) =
         (Map[String, Any]() /: cc.getClass.getDeclaredFields) {(a, f) =>
           f.setAccessible(true)
@@ -480,12 +465,40 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     val utxoRows = movementQuery.filter(_.height_in =!= blockHeight).map(p => (p.transaction_hash, p.address, p.index, p.value, p.height_in)).run
     for ((tx, ad, idx, v, h) <- utxoRows)
       utxoTable += ((Hash(tx) -> idx) -> (Hash(ad), v, h))
+
     utxo.insertAll(utxoRows: _*)
     movementQuery.delete
-
     blockDB.filter(_.block_height === blockHeight).delete
-
     table.close
 
   }
+
+  def getWalletBalances(a: collection.immutable.Set[Hash]): Long = DB withSession { implicit session =>
+    closureBalances.filter(_.address inSetBind a.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
+  }
+
+  def getBalance(a: Hash): Long = DB withSession { implicit session =>
+    balances.filter(_.address === Hash.hashToArray(a)).map(_.balance).firstOption.getOrElse(0L)
+  }
+
+  def getAllBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
+    balances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
+  }
+
+  def getClosures(): collection.immutable.Map[Hash, Hash] = DB withSession { implicit session =>
+    addresses.sortBy(_.hash).map(p=>(p.hash, p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
+  }
+
+  def countClosures(): Int = DB withSession { implicit session =>
+    addresses.groupBy(_.representant).length.run
+  }
+
+  def getWallet(a: Hash): collection.immutable.Set[Hash] = DB withSession { implicit session =>
+    addresses.filter(_.representant === Hash.hashToArray(a)).map(_.hash).run.map(Hash(_)).toSet
+  }
+
+  def getAllWalletBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
+    closureBalances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
+  }
+
 }
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index 317246b..fcca35f 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -8,7 +8,7 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     startDocker should be (0)
   }
 
-  it should "communicate bitcoin data" in {
+  it should "create 103 blocks and 6 txs with bitcoin" in {
     gen(100) should be (0)
     gen(1) should be (0)
     gen(1) should be (0)
@@ -16,51 +16,79 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     gen(1) should be (0)
   }
 
-  "explorer" should "throw postgres exception if there are no blocks" in {
+  it should "read from database" in {
+    bitcoinCount should be (104)
+  }
+
+  it should "read from bitcoin" in {
+    bitcoinCount should be (104)
+  }
+
+  it should "fail sending 5000 bitcoins" in {
+    addTx(5000) should be (6)
+    bitcoinCount should be (104)
+  }
+
+  "explorer" should "throw postgres exception before run" in {
     a [org.postgresql.util.PSQLException] should be thrownBy {
       bgeCount should be (0)
     }
   }
 
   it should "populate blocks" in {
-    savePopulate
-    bgeCount should be (bitcoinCount)
+    savePopulate should be (success)
+    bitcoinCount should be (bgeCount)
   }
 
-  it should "have same stats using create or update" in {
+  it should "resume blocks" in {
     gen(1) should be (0)
-    saveResume should be (None)
-    bgeCount should be (bitcoinCount)
-    gen(1) should be (0)
-  }
-
-  it should "add correclty blocks" in {
-    saveResume should be (None)
-    bgeCount should be (bitcoinCount)
+    saveResume should be (success)
+    bitcoinCount should be (bgeCount)
   }
 
-  it should "add blocks and txs" in {
-    addTx(6) should be (0)
-    gen(1) should be (0)
+  it should "resume more blocks" in {
     gen(1) should be (0)
-    addTx(7) should be (0)
     gen(1) should be (0)
-    addTx(7) should be (0)
-    gen(1) should be (0)
-    saveResume should be (None)
-    bgeCount should be (bitcoinCount)
+    saveResume should be (success)
+    bitcoinCount should be (bgeCount)
+  }
+
+  val MAX = 10
+
+  for (i <- 1 to MAX) {
+
+    val TITLE = s"($i/$MAX)"
+
+    it should s"resume blocks and txs $TITLE" in {
+      addTx(16) should be (0)
+      addTx(8) should be (0)
+      gen(2) should be (0)
+      saveResume should be (success)
+      bitcoinCount should be (bgeCount)
+    }
+
+    it should s"rollback blocks $TITLE" in {
+      val initialStat = stat
+      saveRollback(5)
+      saveResume should be (success)
+      stat should be (initialStat)
+      bitcoinCount should be (bgeCount)
+    }
+
+    it should s"resume more blocks and txs $TITLE" in {
+      addTx(6) should be (0)
+      gen(1) should be (0)
+      gen(1) should be (0)
+      addTx(7) should be (0)
+      gen(1) should be (0)
+      addTx(7) should be (0)
+      gen(1) should be (0)
+      saveResume should be (success)
+      bitcoinCount should be (bgeCount)
+    }
   }
 
-  it should "work after several rollbacks" in {
-    val initialStat = stat
-    rollBackLast
-    saveResume should be (None)
-    bgeCount should be (bitcoinCount)
-    stat should be (initialStat)
-    addTx(8)
-    addTx(8)
-    gen(2) should be (0)
-    saveResume should be (None)
-    bgeCount should be (bitcoinCount)
+  it should s"have generated $bgeCount blocks in the database" in {
+    bitcoinCount should be (bgeCount)
   }
 }
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 090b7f8..5065901 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -2,7 +2,6 @@ import util.Hash
 import Explorer._
 import sys.process._
 
-
 object TestExplorer {
 
   // BITCOIN-CLI functions
@@ -25,7 +24,11 @@ object TestExplorer {
 
   def stat = currentStat
 
-  def rollBackLast = rollBack(blockCount-1)
+  def saveRollback(i: Int) = {
+    for (_ <- 0 until i)
+      rollBack(blockCount-1)
+    populateStats
+  }
 
   def saveResume = {
     val init = currentStat
@@ -42,9 +45,11 @@ object TestExplorer {
     assertBalancesCreatedCorrectly
   }
 
-  def s: collection.immutable.Map[Hash, Long] = collection.immutable.Map()
+  def success = None
+
+  private def s: collection.immutable.Map[Hash, Long] = collection.immutable.Map()
 
-  def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
+  private def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
 
   def assertBalancesCreatedCorrectly =
     assertBalancesUpdatedCorrectly(s,s,s,s,s,x,0,0)
@@ -58,55 +63,55 @@ object TestExplorer {
     changedReps: collection.immutable.Map[Hash, Set[Hash]],
     addedAdds: Int,
     addedReps: Int): Option[String] = {
-
+    
     lazy val ca = changedAddresses - Hash.zero(0)
     lazy val s1 = ca.map(_._2).sum
     lazy val s2 = repsAndChanges.map(_._2).sum
     lazy val b3 = getSumBalance
     lazy val b4 = getSumWalletsBalance
-    lazy val message2 = s"""
-      WALLETSs and CHANGES ${str1(repsAndChanges)}
-      ADDRESSs and CHANGES ${str1(ca)}
-      ADDED REPS ${str1(repsAndBalances)}
-      REPS TABLE  ${str2(getClosures())}
-      ADDs BALANCES ${str1(getAllBalances())}
-      REPS BALANCES ${str1(getAllWalletBalances())}
-      WALLETS CHANGED MORE THAN ADDRESSES ###
-        ${nr(s2-s1)} btcs
-      WALLETs TABLES MORE THAN ADDRESSES ###
-        ${nr(b4-b3)} btcs
-      UTXO / ADDRESS / WALLET
-        ${nr(getUTXOSBalance())} / ${nr(b3)} / ${nr(b4)}
-      ${changedReps.map(p=> p._1.toString.drop(2).take(4)+" -> " + p._2.map(_.toString.drop(2).take(4)))}
-"""
-    lazy val message = ""
-
-    // test updateBalances
-    if (adsAndBalances.exists(r => r._2 < 0))
-      Some(message + "  NEGATIVE ADDRESS - updateBalances")
-    else if(repsAndBalances.exists(r => r._2 < 0))
-      Some(message + "  NEGATIVE WALLET - updateBalances")
-    else if (repsAndBalances.exists(r => r._1 != getRepresentant(r._1)))
-      Some(message + "  WRONG UPDATE - updateBalances")
+    lazy val ut = getUTXOSBalance()
+    lazy val ar = addedReps
+      - (changedReps.values.foldLeft(Set[Hash]())((s,p) => s++p)--changedReps.keys).size
+      + changedReps.keys.size
+
+    lazy val negativeAddressOption = adsAndBalances.filter(_._2 < 0).map(p => (pri(p._1), nr(p._2))).headOption
+    lazy val negativeWalletOption = repsAndBalances.filter(_._2 < 0).headOption
+    lazy val balanceResults = s"UTXOs ${nr(ut)} ADDs ${nr(b3)} WALLETs ${nr(b4)}"
+    lazy val t1 = negativeWalletOption.get
+    lazy val negativeWalletString = s"REP: ${pri(getRepresentant(t1._1))}  WALL: ${pri(t1._1)}  BTC ${nr(t1._2)} ${getWallet(t1._1)}"
+
+    // test updateStatistics && test updateBalances
+    if (repsAndBalances.exists(r => r._1 != getRepresentant(r._1)))
+      Some(s"___ WRONG UPDATE ___")
     else if (s1 != s2)
-      Some(message + "  WRONG CHANGED VALUES - updateBalances")
-    else if (b3 != b4)
-      Some(message  +"  WRONG CHANGED BALANCE - updateBalances")
-    // test updateStatistics
-    else if (addedReps <= changedReps.values.map(_.size).sum)
-      Some(message + "  LOST WALLETS - updateStatistics")
+      Some(s"___ WRONG CHANGED VALUES ___")
     else if (addedAdds < 0)
-      Some(message + "  LOST ADDRESSES - updateStatistics")
+      Some(s"___ LOST ADDRESSES ___")
+    else if (negativeAddressOption != None)
+      Some(s"___ NEGATIVE ADDRESS ___")
+    else if (b3 != b4)
+      Some(s"___ WRONG WALLET BALANCE ___")
+    else if (ut != b4)
+      Some(s"___ WRONG ADDRESS BALANCE ___")
+    else if (ar < 0)
+      Some(s"___ LOST $ar REPRESENTANTS ___ ")
+    else if (negativeWalletOption != None)
+      Some(s"___ NEGATIVE WALLET ___ $negativeWalletString")
     else
       None
-
   }
 
   // Some help formatter
-  def nr(b: Long): String = ((b/10000)/10000.0).toString
+  private def nr(b: Long): String = ((b/10000)/10000.0).toString
+
+  private def sx = "\n      "
+
+  private def pri(a: Hash) = a.toString.drop(2).take(4)
+
+  private def str1(n: collection.immutable.Map[Hash, Long]): String = "#: " + n.size + "  total: " + nr(n.map(_._2).sum) + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
 
-  def str1(n: collection.immutable.Map[Hash, Long]): String = if (n.size> 0 && n.size < 25) "#: " + n.size + "  total: " + nr(n.map(_._2).sum) +"\n    " + n.toSeq.sortBy(_._1.toString.drop(2).take(4)).map(m=>m._1.toString.drop(2).take(4)+": "+(nr(m._2))).mkString("\n    ") else n.size.toString
+  private def str2(n: collection.immutable.Map[Hash, Hash]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) +" => "+(pri(m._2))).mkString(sx)
 
-  def str2(n: collection.immutable.Map[Hash, Hash]): String = if (n.size > 0 && n.size < 25) "(" + n.size + ")\n    " + n.toSeq.sortBy(_._1.toString.drop(2).take(4)).map(m=>m._1.toString.drop(2).take(4)+" => "+(m._2.toString.drop(2).take(4))).mkString("\n    ") else n.size.toString
+  private def str3(n: collection.immutable.Map[Hash, Set[Hash]]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) + " => "+ m._2.map((pri(_))).mkString(" - ")).mkString(sx)
 
 }

From 9741baa0c8c56fae04a620db78ddd5b99cedf136 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 1 Jun 2018 17:49:43 +0200
Subject: [PATCH 49/69] Delete useless info log

---
 build.sbt                                     |  2 +
 src/main/scala/bge/Explorer.scala             |  8 +-
 .../scala/bge/actions/PopulateClosure.scala   |  6 +-
 .../scala/bge/actions/ResumeBlockReader.scala |  5 +-
 .../scala/bge/actions/ResumeClosure.scala     |  6 --
 src/main/scala/bge/core/AddressClosure.scala  | 10 +--
 src/main/scala/bge/core/FastBlockReader.scala | 10 +--
 src/main/scala/bge/core/PeerSource.scala      |  2 +-
 src/main/scala/db/BitcoinDB.scala             | 80 +++++++++----------
 src/main/scala/util/LmdbMap.scala             |  1 -
 src/main/scala/util/package.scala             |  7 +-
 src/test/scala/ExplorerIntegrationTests.scala | 50 ++++++------
 src/test/scala/TestExplorer.scala             |  7 +-
 13 files changed, 91 insertions(+), 103 deletions(-)

diff --git a/build.sbt b/build.sbt
index a353241..1c4c537 100644
--- a/build.sbt
+++ b/build.sbt
@@ -96,3 +96,5 @@ javaOptions in run += "-Xmx32G"
 javaOptions in run += "-Xms16"
 
 fork := true
+
+testOptions in Test += Tests.Argument("-oD")
diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 63d42b9..2de7b2c 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -118,7 +118,7 @@ object Explorer extends App with db.BitcoinDB {
     val checkBalances =  getSumBalance == getSumWalletsBalance
 
     if (!checkBalances)
-      log.info("Error updating balances .... create it again")
+      log.info("Error during update of balances or stats .... creating balances and stats")
 
     if (read.changedAddresses.size < balanceUpdateLimit && checkBalances) {
       val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
@@ -153,12 +153,12 @@ object Explorer extends App with db.BitcoinDB {
     while (new java.io.File(lockFile).exists) {
       if (blockCount > chain.getBestChainHeight) {
         log.info("waiting for new blocks")
-        waitForBitcoinJBlock(blockCount) //wait until the chain overtakes our DB
+        waitForBitcoinJBlock(blockCount) // wait until the chain overtakes our DB
       }
       resume
     }
 
-    log.info("process stopped")
+    log.info("Look file deleted. BGE shut down correctly!")
   }
 
   def getWrongBlock: Option[Int] = {
@@ -189,8 +189,6 @@ object Explorer extends App with db.BitcoinDB {
   def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int):
       (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
 
-    log.info(changedAddresses.size + " addresses changed balance")
-
     val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = updateBalanceTables(changedAddresses.toMap, changedReps.toMap)
     insertRichestAddresses
     insertRichestClosures
diff --git a/src/main/scala/bge/actions/PopulateClosure.scala b/src/main/scala/bge/actions/PopulateClosure.scala
index 9754b58..899b895 100644
--- a/src/main/scala/bge/actions/PopulateClosure.scala
+++ b/src/main/scala/bge/actions/PopulateClosure.scala
@@ -19,7 +19,7 @@ class PopulateClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHei
     var counter = 0
     var counterTotal = 0
 
-    log.info("Saving tree to database...")
+//    log.info("Saving tree to database...")
     var counterFinal = 0
     // tree.elements.keys.foldLeft(tree){(t,value) =>
     //   val (parentOption, newTree) = tree.find(value)
@@ -42,12 +42,12 @@ class PopulateClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHei
       }
       if (counterFinal % 1000000 == 0) {
         counterFinal = 0
-        log.info("Saved until element %s in %s s, %s µs per element" format (counterTotal, (System.currentTimeMillis - timeStart)/1000, (System.currentTimeMillis - timeStart)*1000/(counterTotal+1)))
+        log.info("Saved to DB until element %s in %s s, %s µs per element" format (counterTotal, (System.currentTimeMillis - timeStart)/1000, (System.currentTimeMillis - timeStart)*1000/(counterTotal+1)))
       }
       // newTree
     }
 
-    log.info("Saved until element %s in %s s, %s µs per element" format (counterTotal, (System.currentTimeMillis - timeStart)/1000, (System.currentTimeMillis - timeStart)*1000/(counterTotal+1)))
+    log.info("Saved to DB until element %s in %s s, %s µs per element" format (counterTotal, (System.currentTimeMillis - timeStart)/1000, (System.currentTimeMillis - timeStart)*1000/(counterTotal+1)))
 
     saveElementsToDatabase(queries, counter)
 
diff --git a/src/main/scala/bge/actions/ResumeBlockReader.scala b/src/main/scala/bge/actions/ResumeBlockReader.scala
index e676a4e..a1febe1 100644
--- a/src/main/scala/bge/actions/ResumeBlockReader.scala
+++ b/src/main/scala/bge/actions/ResumeBlockReader.scala
@@ -92,7 +92,7 @@ class ResumeBlockReader extends FastBlockReader with PeerSource {
   }
 
   override def post = {
-    log.info("finishing ...")
+    //log.info("finishing ...")
     super.post
     table.close
   }
@@ -108,7 +108,6 @@ class ResumeBlockReader extends FastBlockReader with PeerSource {
     }
 
   def saveUTXOs = {
-    log.info("Inserting UTXOs into SQL database ...")
 
     def vectorUTXOConverter[A, B, C](v: Map[(Hash, A), (Hash, B, C)]) = v map {
       case ((a, b), (c, d, e)) => (hashToArray(a), hashToArray(c), b, d, e)
@@ -120,7 +119,5 @@ class ResumeBlockReader extends FastBlockReader with PeerSource {
 
     newUtxos.clear
 
-    log.info("Data inserted")
-
   }
 }
diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index aa34984..0c85a46 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -82,9 +82,6 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
   }
 
   def saveAddresses = {
-
-    log.info("Inserting Addresses into SQL database ...")
-
     val convertedVector = addressBuffer map (p => (hashToArray(p._1), hashToArray(p._2)))
 
     try{
@@ -97,9 +94,6 @@ class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeigh
     addedAds += addressBuffer.size
 
     addressBuffer.clear
-
-    log.info("Data inserted")
-
   }
 }
 
diff --git a/src/main/scala/bge/core/AddressClosure.scala b/src/main/scala/bge/core/AddressClosure.scala
index f0e3173..810603f 100644
--- a/src/main/scala/bge/core/AddressClosure.scala
+++ b/src/main/scala/bge/core/AddressClosure.scala
@@ -32,18 +32,18 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
     def addBlocks(startIndex: Int, tree: DisjointSets[Hash]): DisjointSets[Hash] = {
       val blocks = blockHeights.slice(startIndex,startIndex+closureReadSize)
       for (blockNo <- blocks.headOption)
-      log.info("reading " + blocks.length + " blocks from " + blockNo)
+      log.info("Closure " + blocks.length + " blocks from block " + blockNo)
       val txAndAddressList = txListQuery(blocks)
       val addressesPerTxMap = txAndAddressList.groupBy(p=>Hash(p._1))
       val hashList = addressesPerTxMap.values map (_ map (p=>Hash(p._2)))
       val nontrivials = hashList filter (_.length > 1)
 
-      log.info("folding and merging " + nontrivials.size)
+//      log.info("folding and merging " + nontrivials.size)
       nontrivials.foldLeft (tree) ((t,l) => insertInputsIntoTree(l,t))
     }
 
     val result = (0 until blockHeights.length by closureReadSize).foldRight(new DisjointSets[Hash](unionFindTable))(addBlocks)
-    log.info("finished generation")
+    //log.info("finished generation")
     result
   }
 
@@ -53,7 +53,7 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
     addedTree.union(addresses)
   }
 
-  log.info("applying closure ")
+//  log.info("applying closure ")
   val timeStart = System.currentTimeMillis
   val startTableSize = unionFindTable.size
   val countSave = saveTree(generateTree)
@@ -63,5 +63,3 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
   log.info("Total of %s addresses added to closures in %s s, %s µs per address" format
     (countSave, totalTime / 1000, 1000 * totalTime / (countSave + 1)))
 }
-
-
diff --git a/src/main/scala/bge/core/FastBlockReader.scala b/src/main/scala/bge/core/FastBlockReader.scala
index 48119d3..2120eb0 100644
--- a/src/main/scala/bge/core/FastBlockReader.scala
+++ b/src/main/scala/bge/core/FastBlockReader.scala
@@ -73,7 +73,7 @@ trait FastBlockReader extends BlockReader {
   def finishBlock(b: Hash, txs: Int, btcs: Long, tstamp: Long, height:Int) = {
     processedBlocks :+= height
     insertBlock(b, height, txs, btcs, tstamp)
-    //log.info("Saved block " + height + " consisting of " + txs + " txs")
+    log.info(s"Saved block $height consisting of $txs txs")
   }
 
   def pre  = {
@@ -102,9 +102,8 @@ trait FastBlockReader extends BlockReader {
   def saveDataToDB: Unit =
   {
     val amount = vectorBlocks.length + vectorMovements.length
-    log.info("Saving blocks/movements (" + amount + ")  into database ...")
-
     val convertedVectorBlocks = vectorBlocks map { case (a,b,c,d,e) => (a.array.toArray,b,c,d,e) }
+
     DB.withSession(blockDB.insertAll(convertedVectorBlocks:_*)(_))
 
     def ohc(e:Option[Hash]):Array[Byte] = e.getOrElse(Hash.zero(0)).array.toArray
@@ -120,11 +119,8 @@ trait FastBlockReader extends BlockReader {
         throw e.getNextException
     }
 
-
     vectorMovements = Vector()
     vectorBlocks = Vector()
-
-    //log.info("Data inserted")
   }
 
   // block
@@ -179,6 +175,4 @@ trait FastBlockReader extends BlockReader {
   def removeUTXO(outpointTransactionHash: util.Hash, outpointIndex: Int): UTXOs = {
     outputMap -= (outpointTransactionHash -> outpointIndex)
   }
-
-
 }
diff --git a/src/main/scala/bge/core/PeerSource.scala b/src/main/scala/bge/core/PeerSource.scala
index a500de1..4ab2918 100644
--- a/src/main/scala/bge/core/PeerSource.scala
+++ b/src/main/scala/bge/core/PeerSource.scala
@@ -11,7 +11,7 @@ trait PeerSource extends BlockSource {
   {
     val peer = peerGroup.getConnectedPeers().get(0);
     for ((_,end) <- truncated.lastOption)
-      log.info("reading blocks from " + blockCount + " to " + end)
+      log.info("Reading blocks from " + blockCount + " to " + end)
 
     for ((blockHash,no) <- truncated.toIterator) yield {
       val future = peer.getBlock(blockHash)
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 30dd3dd..bdd4b33 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -104,7 +104,7 @@ trait BitcoinDB {
   def createBalanceTables = {
     var clock = System.currentTimeMillis
     DB withSession { implicit session =>
-      log.info("Creating balances")
+      //log.info("Creating balances")
       deleteIfExists(balances, closureBalances)
       balances.ddl.create
       closureBalances.ddl.create
@@ -145,19 +145,6 @@ trait BitcoinDB {
     }
   }
 
-  def updateAdsBalancesTable[A <: Table[(Array[Byte], Long)] with AddressField](adsAndBalances: scala.collection.immutable.Map[Hash, Long], balances: TableQuery[A]): Unit = {
-    DB withSession { implicit session =>
-      for {
-        (address, balance) <- adsAndBalances
-        addressArray = Hash.hashToArray(address)
-      }
-      if (balance != 0L)
-        balances.insertOrUpdate(addressArray, balance)
-      else
-        balances.filter(_.address === addressArray).delete
-    }
-  }
-
   def getSumBalance: Long = DB withSession { implicit session =>
     balances.map(_.balance).sum.run.getOrElse(0)
   }
@@ -171,34 +158,37 @@ trait BitcoinDB {
   }
 
   def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit = {
-
-    DB withSession { implicit session =>
+    DB withTransaction { implicit session =>
       // delete merged wallets
       val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ repsAndBalances.keys ++ changedReps.keys).map(Hash.hashToArray)
       closureBalances.filter(_.address inSetBind toDelete).delete
+      for {
+        (balances, table) <- Set((adsAndBalances, balances), (repsAndBalances, closureBalances))
+        (address, balance) <- balances
+      } {
+        if (balance != 0L)
+          table.insertOrUpdate(Hash.hashToArray(address), balance)
+        else
+          table.filter(_.address === Hash.hashToArray(address)).delete
+      }
     }
-    updateAdsBalancesTable(adsAndBalances, balances)
-    updateAdsBalancesTable(repsAndBalances, closureBalances)
   }
 
-  // Unsave functions, use only for testing
-  // end of unsave functions block
   def getRepresentant(a: Hash): Hash = DB withSession { implicit session =>
     addresses.filter(_.hash === Hash.hashToArray(a)).map(_.representant).run.map(Hash(_)).headOption.getOrElse(a)
   }
 
-
-def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]):
+  def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]):
     (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long], collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
     val clock = System.currentTimeMillis
-    log.info("Updating balances ...")
+    //log.info(s"Updating ${changedAddresses.size} balances ...")
     currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
 
     val adsAndBalances: scala.collection.immutable.Map[Hash, Long] =
       for ((address, change) <- changedAddresses - Hash.zero(0))
          yield (address, getBalance(address) + change)
 
-    // FIXME refactor
+    // FIXME it produces negative balances and split closures
     val repsAndChanges: collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
       .distinct
       .map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
@@ -211,6 +201,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
       .sortBy(_._1)
       .toMap
 
+  // FIXME both maps are wrong
     val repsAndAvailable: scala.collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
       .distinct
       .map(p => (p,
@@ -237,19 +228,19 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
   }
 
   def insertRichestClosures = {
-    log.info("Calculating richest closure list...")
+    //log.info("Calculating richest closure list...")
     var startTime = System.currentTimeMillis
     DB withSession { implicit session =>
       val bh = blockCount - 1
       val topClosures = closureBalances.sortBy(_.balance.desc).take(richlistSize).run.toVector
       val topClosuresWithBh = for ((rep, bal) <- topClosures) yield (bh, rep, bal)
       richestClosures.insertAll(topClosuresWithBh: _*)
-      log.info("RichestList calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s")
+      log.info("Richest wallets calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s")
     }
   }
 
   def insertRichestAddresses = {
-    log.info("Calculating richest address list...")
+//    log.info("Calculating richest address list...")
     var startTime = System.currentTimeMillis
     DB withSession { implicit session =>
       Q.updateNA("""
@@ -266,7 +257,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
         balance desc
       limit """ + richlistSize + """
     ;""").execute
-      log.info("RichestList calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s")
+      log.info("Richest addresses calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s")
     }
   }
 
@@ -276,7 +267,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
 
-    log.info("Calculating stats...")
+//    log.info("Calculating stats...")
 
     val startTime = System.currentTimeMillis
     DB withSession { implicit session =>
@@ -299,14 +290,14 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
        """ + (System.currentTimeMillis / 1000).toString + """;"""
 
       (Q.u + query).execute
-      log.info("Stats calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s");
+      log.info("Stat calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s");
 
     }
   }
 
   def updateStatistics(changedReps: Map[Hash, Set[Hash]], addedAds: Int, addedReps: Int) = {
 
-    log.info("Updating stats")
+  //  log.info("Updating stats")
 
     val time = System.currentTimeMillis
     val (nonDustAddresses, addressGini) = getGini(balances)
@@ -328,14 +319,14 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
       stat.total_transactions = blockDB.map(_.txs).filter(_ > 0).sum.run.getOrElse(0).toLong
       stat.total_bitcoins_in_addresses = balances.map(_.balance).sum.run.getOrElse(0L) / 100000000
       stat.total_addresses += addedAds
-      stat.total_closures += addedClosures
+      stat.total_closures += addedClosures // FIXME after a rollback it seems to be wrong.
       saveStat(stat)
-      log.info("Updated in " + (System.currentTimeMillis - time) / 1000 + " seconds")
+      log.info("Stat updated in " + (System.currentTimeMillis - time) / 1000 + " seconds")
     }
   }
 
   def getGini[A <: Table[_] with BalanceField](balanceTable: TableQuery[A]): (Long, Double) = {
-    log.info("calculating Gini: " + balanceTable)
+    //log.info("calculating Gini: " + balanceTable)
     val time = System.currentTimeMillis
 
     val balanceVector = DB withSession { implicit session =>
@@ -349,7 +340,7 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
     val summe = balances.sum
     val mainSum = balances.zipWithIndex.map(p => p._1 * (p._2 + 1.0) / n).sum
     val gini: Double = if (n == 0) 0.0 else 2.0 * mainSum / (summe) - (n + 1.0) / n
-    log.info("gini calculated in " + (System.currentTimeMillis - time) / 1000 + "s")
+    //log.info("Gini calculated in " + (System.currentTimeMillis - time) / 1000 + "s")
     (n, gini)
   }
 
@@ -424,19 +415,23 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
                   var total_closures_no_dust: Long,
                   var gini_closure: Double,
                   var gini_address: Double,
-                  var tstamp: Long) { override def equals(that: Any): Boolean = {
-      def getCCParams(cc: AnyRef) =
-        (Map[String, Any]() /: cc.getClass.getDeclaredFields) {(a, f) =>
-          f.setAccessible(true)
-          a + (f.getName -> f.get(cc))
-        }
+                  var tstamp: Long)  {
+
+    def toMap() = getCCParams(this)
+
+    override def equals(that: Any): Boolean = {
       that match {
         case that: Stat => 
-          getCCParams(this)-"tstamp" == getCCParams(that)-"tstamp"
+          this.toMap()-"tstamp" == that.toMap()-"tstamp"
         case _ =>
           false
       }
     }
+
+    override def toString(): String = {
+      getCCParams(this).toString()
+    }
+
   }
 
   def lastCompletedHeight: Int = DB withSession { implicit session =>
@@ -500,5 +495,4 @@ def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long],
   def getAllWalletBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
     closureBalances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
   }
-
 }
diff --git a/src/main/scala/util/LmdbMap.scala b/src/main/scala/util/LmdbMap.scala
index a386216..203700a 100644
--- a/src/main/scala/util/LmdbMap.scala
+++ b/src/main/scala/util/LmdbMap.scala
@@ -128,7 +128,6 @@ class LmdbMap(val name: String = java.util.UUID.randomUUID.toString)
     for (kv <- cache)
       db.put(tx, kv._1, kv._2)
     tx.commit
-    log.info("commit took " + (System.currentTimeMillis - t) + " ms")
     cache.clear
   }
 
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index b2fbc4a..c114db7 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -99,8 +99,13 @@ package object util
     }
   }
 
+  // convert case class to map.
+  def getCCParams(cc: AnyRef) =
+    (Map[String, Any]() /: cc.getClass.getDeclaredFields) {(a, f) =>
+      f.setAccessible(true)
+      a + (f.getName -> f.get(cc))
+    }
 
-  
   def convertToMap(a: DisjointSets[Hash]): collection.mutable.Map[Hash, Set[Hash]] = {
     val t: collection.mutable.Map[Hash, Set[Hash]] = collection.mutable.Map.empty
     for {
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index fcca35f..64bcc6f 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -4,15 +4,17 @@ import org.scalatest._
 
 class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
+  val RUNS = 2
+
   "docker" should "start correctly" in {
     startDocker should be (0)
   }
 
-  it should "create 103 blocks and 6 txs with bitcoin" in {
-    gen(100) should be (0)
-    gen(1) should be (0)
+  it should "create the first 103 blocks and 1 txs with bitcoin" in {
     gen(1) should be (0)
-    addTx(6) should be (0)
+    gen(1) should be (0)   
+    gen(100) should be (0)
+    addTx(95) should be (0)
     gen(1) should be (0)
   }
 
@@ -35,39 +37,37 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     }
   }
 
-  it should "populate blocks" in {
+  it should s"populate $bitcoinCount blocks" in {
     savePopulate should be (success)
     bitcoinCount should be (bgeCount)
   }
 
-  it should "resume blocks" in {
+  it should "resume 1 more block with 0 txs" in {
     gen(1) should be (0)
     saveResume should be (success)
     bitcoinCount should be (bgeCount)
   }
 
-  it should "resume more blocks" in {
-    gen(1) should be (0)
+  it should "resume 1 block with 1 tx " in {
+    addTx(77) should be (0)
     gen(1) should be (0)
     saveResume should be (success)
     bitcoinCount should be (bgeCount)
   }
 
-  val MAX = 10
+  for (i <- 1 to RUNS) {
 
-  for (i <- 1 to MAX) {
+    val TITLE = (if (RUNS == 1) "" else s" ($i/$RUNS)")
 
-    val TITLE = s"($i/$MAX)"
-
-    it should s"resume blocks and txs $TITLE" in {
-      addTx(16) should be (0)
-      addTx(8) should be (0)
-      gen(2) should be (0)
+    it should s"resume 1 block with 2 txs$TITLE" in {
+      addTx(84) should be (0)
+      addTx(71) should be (0)
+      gen(1) should be (0)
       saveResume should be (success)
       bitcoinCount should be (bgeCount)
     }
 
-    it should s"rollback blocks $TITLE" in {
+    it should s"stat are the same after rollback 5 blocks and resuming it again$TITLE" in {
       val initialStat = stat
       saveRollback(5)
       saveResume should be (success)
@@ -75,13 +75,17 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
       bitcoinCount should be (bgeCount)
     }
 
-    it should s"resume more blocks and txs $TITLE" in {
-      addTx(6) should be (0)
-      gen(1) should be (0)
-      gen(1) should be (0)
-      addTx(7) should be (0)
+    it should s"resume 2 blocks with 4 and 5 txs$TITLE" in {
+      addTx(51) should be (0)
+      addTx(68) should be (0)
+      addTx(83) should be (0)
+      addTx(71) should be (0)
       gen(1) should be (0)
-      addTx(7) should be (0)
+      addTx(61) should be (0)
+      addTx(37) should be (0)
+      addTx(59) should be (0)
+      addTx(83) should be (0)
+      addTx(27) should be (0)
       gen(1) should be (0)
       saveResume should be (success)
       bitcoinCount should be (bgeCount)
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 5065901..fa11647 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -70,6 +70,7 @@ object TestExplorer {
     lazy val b3 = getSumBalance
     lazy val b4 = getSumWalletsBalance
     lazy val ut = getUTXOSBalance()
+    lazy val wb = getAllWalletBalances.filter(p=>getWallet(t1._1) contains p._1)
     lazy val ar = addedReps
       - (changedReps.values.foldLeft(Set[Hash]())((s,p) => s++p)--changedReps.keys).size
       + changedReps.keys.size
@@ -78,7 +79,7 @@ object TestExplorer {
     lazy val negativeWalletOption = repsAndBalances.filter(_._2 < 0).headOption
     lazy val balanceResults = s"UTXOs ${nr(ut)} ADDs ${nr(b3)} WALLETs ${nr(b4)}"
     lazy val t1 = negativeWalletOption.get
-    lazy val negativeWalletString = s"REP: ${pri(getRepresentant(t1._1))}  WALL: ${pri(t1._1)}  BTC ${nr(t1._2)} ${getWallet(t1._1)}"
+    lazy val negativeWalletString = s"\nWALLET BALANCE: ${str1(wb)} \nCHANGED REPS: ${str3(changedReps)} \nREPS AND BALANCES ${str1(repsAndBalances.filter(p=>getWallet(t1._1) contains p._1))}\n"
 
     // test updateStatistics && test updateBalances
     if (repsAndBalances.exists(r => r._1 != getRepresentant(r._1)))
@@ -106,9 +107,11 @@ object TestExplorer {
 
   private def sx = "\n      "
 
+  private def sp = " "
+
   private def pri(a: Hash) = a.toString.drop(2).take(4)
 
-  private def str1(n: collection.immutable.Map[Hash, Long]): String = "#: " + n.size + "  total: " + nr(n.map(_._2).sum) + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
+  private def str1(n: collection.immutable.Map[Hash, Long]): String = "(" + n.size + ") total: " + nr(n.map(_._2).sum) + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
 
   private def str2(n: collection.immutable.Map[Hash, Hash]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) +" => "+(pri(m._2))).mkString(sx)
 

From 745ab03098a50290b2830a44b8ae78092db73dec Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Fri, 1 Jun 2018 18:09:34 +0200
Subject: [PATCH 50/69] Remove support for bitcoinJ raw file reader: peer
 reader is fast enought!

---
 .../bge/core/BitcoinDRawFileBlockSource.scala |  32 ----
 .../bge/core/RegTestBlockFileLoader.java      | 176 ------------------
 .../bge/core/TestNetBlockFileLoader.java      | 176 ------------------
 src/main/scala/util/package.scala             |  16 +-
 4 files changed, 2 insertions(+), 398 deletions(-)
 delete mode 100644 src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
 delete mode 100644 src/main/scala/bge/core/RegTestBlockFileLoader.java
 delete mode 100644 src/main/scala/bge/core/TestNetBlockFileLoader.java

diff --git a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala b/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
deleted file mode 100644
index 8386c9b..0000000
--- a/src/main/scala/bge/core/BitcoinDRawFileBlockSource.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-package core
-
-import org.bitcoinj.core._
-import util._
-
-import scala.collection.JavaConverters.asScalaIterator
-
-// In java that should be implements libs.BlockSource
-trait BitcoinDRawFileBlockSource extends BlockSource
-{
-  override def blockSource: Iterator[(Block,Int)] = {
-
-    startBitcoinJ
-    log.info("starting")
-    val almostCurrentChain =
-      if (maxPopulate == 0) getCurrentLongestChainFromBlockCount
-      else if (maxPopulate > 0)
-        getCurrentLongestChainFromBlockCount.take(maxPopulate)
-      else throw new Exception(s"invalid parameter bitcoin.maxPopulate = '$maxPopulate'")
-
-    val blockMap = almostCurrentChain toMap
-
-    for {
-      block <- asScalaIterator(loader)
-      hash = block.getHash
-      if blockMap.contains(hash)      
-    }
-    yield 
-      (block, blockMap(hash))
-  }
-  
-}
diff --git a/src/main/scala/bge/core/RegTestBlockFileLoader.java b/src/main/scala/bge/core/RegTestBlockFileLoader.java
deleted file mode 100644
index 21e6bbf..0000000
--- a/src/main/scala/bge/core/RegTestBlockFileLoader.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright 2012 Matt Corallo.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package util;
-
-import org.bitcoinj.core.Block;
-import org.bitcoinj.core.NetworkParameters;
-import org.bitcoinj.core.ProtocolException;
-import org.bitcoinj.core.Utils;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.NoSuchElementException;
-
-/**
- * <p>This class reads block files stored in the Bitcoin Core format. This is simply a way to concatenate
- * blocks together. Importing block data with this tool can be a lot faster than syncing over the network, if you
- * have the files available.</p>
- * 
- * <p>In order to comply with Iterator&lt;Block>, this class swallows a lot of IOExceptions, which may result in a few
- * blocks being missed followed by a huge set of orphan blocks.</p>
- * 
- * <p>To blindly import all files which can be found in Bitcoin Core (version >= 0.8) datadir automatically,
- * try this code fragment:<br>
- * BlockFileLoader loader = new BlockFileLoader(BlockFileLoader.getReferenceClientBlockFileList());<br>
- * for (Block block : loader) {<br>
- * &nbsp;&nbsp;try { chain.add(block); } catch (Exception e) { }<br>
- * }</p>
- */
-public class RegTestBlockFileLoader implements Iterable<Block>, Iterator<Block> {
-    /**
-     * Gets the list of files which contain blocks from Bitcoin Core.
-     */
-    public static List<File> getReferenceClientBlockFileList() {
-        String defaultDataDir;
-        String OS = System.getProperty("os.name").toLowerCase();
-        if (OS.indexOf("win") >= 0) {
-            defaultDataDir = System.getenv("APPDATA") + "\\.bitcoin\\regtest\blocks\\";
-        } else if (OS.indexOf("mac") >= 0 || (OS.indexOf("darwin") >= 0)) {
-            defaultDataDir = System.getProperty("user.home") + "/Library/Application Support/Bitcoin/regtest/blocks/";
-        } else {
-            defaultDataDir = System.getProperty("user.home") + "/.bitcoin/regtest/blocks/";
-        }
-        
-        List<File> list = new LinkedList<>();
-        for (int i = 0; true; i++) {
-            File file = new File(defaultDataDir + String.format(Locale.US, "blk%05d.dat", i));
-            if (!file.exists())
-                break;
-            list.add(file);
-        }
-        return list;
-    }
-    
-    private Iterator<File> fileIt;
-    private FileInputStream currentFileStream = null;
-    private Block nextBlock = null;
-    private NetworkParameters params;
-    
-    public RegTestBlockFileLoader(NetworkParameters params, List<File> files) {
-        fileIt = files.iterator();
-        this.params = params;
-    }
-    
-    @Override
-    public boolean hasNext() {
-        if (nextBlock == null)
-            loadNextBlock();
-        return nextBlock != null;
-    }
-
-    @Override
-    public Block next() throws NoSuchElementException {
-        if (!hasNext())
-            throw new NoSuchElementException();
-        Block next = nextBlock;
-        nextBlock = null;
-        return next;
-    }
-    
-    private void loadNextBlock() {
-        while (true) {
-            try {
-                if (!fileIt.hasNext() && (currentFileStream == null || currentFileStream.available() < 1))
-                    break;
-            } catch (IOException e) {
-                currentFileStream = null;
-                if (!fileIt.hasNext())
-                    break;
-            }
-            while (true) {
-                try {
-                    if (currentFileStream != null && currentFileStream.available() > 0)
-                        break;
-                } catch (IOException e1) {
-                    currentFileStream = null;
-                }
-                if (!fileIt.hasNext()) {
-                    nextBlock = null;
-                    currentFileStream = null;
-                    return;
-                }
-                try {
-                    currentFileStream = new FileInputStream(fileIt.next());
-                } catch (FileNotFoundException e) {
-                    currentFileStream = null;
-                }
-            }
-            try {
-                int nextChar = currentFileStream.read();
-                while (nextChar != -1) {
-                    if (nextChar != ((params.getPacketMagic() >>> 24) & 0xff)) {
-                        nextChar = currentFileStream.read();
-                        continue;
-                    }
-                    nextChar = currentFileStream.read();
-                    if (nextChar != ((params.getPacketMagic() >>> 16) & 0xff))
-                        continue;
-                    nextChar = currentFileStream.read();
-                    if (nextChar != ((params.getPacketMagic() >>> 8) & 0xff))
-                        continue;
-                    nextChar = currentFileStream.read();
-                    if (nextChar == (params.getPacketMagic() & 0xff))
-                        break;
-                }
-                byte[] bytes = new byte[4];
-                currentFileStream.read(bytes, 0, 4);
-                long size = Utils.readUint32BE(Utils.reverseBytes(bytes), 0);
-                // We allow larger than MAX_BLOCK_SIZE because test code uses this as well.
-                if (size > Block.MAX_BLOCK_SIZE*2 || size <= 0)
-                    continue;
-                bytes = new byte[(int) size];
-                currentFileStream.read(bytes, 0, (int) size);
-                try {
-                    nextBlock = params.getDefaultSerializer().makeBlock(bytes);
-                } catch (ProtocolException e) {
-                    nextBlock = null;
-                    continue;
-                }
-                break;
-            } catch (IOException e) {
-                currentFileStream = null;
-                continue;
-            }
-        }
-    }
-
-    @Override
-    public void remove() throws UnsupportedOperationException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Iterator<Block> iterator() {
-        return this;
-    }
-}
diff --git a/src/main/scala/bge/core/TestNetBlockFileLoader.java b/src/main/scala/bge/core/TestNetBlockFileLoader.java
deleted file mode 100644
index b70d2fa..0000000
--- a/src/main/scala/bge/core/TestNetBlockFileLoader.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/*
- * Copyright 2012 Matt Corallo.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package util;
-
-import org.bitcoinj.core.Block;
-import org.bitcoinj.core.NetworkParameters;
-import org.bitcoinj.core.ProtocolException;
-import org.bitcoinj.core.Utils;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-import java.util.NoSuchElementException;
-
-/**
- * <p>This class reads block files stored in the Bitcoin Core format. This is simply a way to concatenate
- * blocks together. Importing block data with this tool can be a lot faster than syncing over the network, if you
- * have the files available.</p>
- * 
- * <p>In order to comply with Iterator&lt;Block>, this class swallows a lot of IOExceptions, which may result in a few
- * blocks being missed followed by a huge set of orphan blocks.</p>
- * 
- * <p>To blindly import all files which can be found in Bitcoin Core (version >= 0.8) datadir automatically,
- * try this code fragment:<br>
- * BlockFileLoader loader = new BlockFileLoader(BlockFileLoader.getReferenceClientBlockFileList());<br>
- * for (Block block : loader) {<br>
- * &nbsp;&nbsp;try { chain.add(block); } catch (Exception e) { }<br>
- * }</p>
- */
-public class TestNetBlockFileLoader implements Iterable<Block>, Iterator<Block> {
-    /**
-     * Gets the list of files which contain blocks from Bitcoin Core.
-     */
-    public static List<File> getReferenceClientBlockFileList() {
-        String defaultDataDir;
-        String OS = System.getProperty("os.name").toLowerCase();
-        if (OS.indexOf("win") >= 0) {
-            defaultDataDir = System.getenv("APPDATA") + "\\.bitcoin\\blocks\\";
-        } else if (OS.indexOf("mac") >= 0 || (OS.indexOf("darwin") >= 0)) {
-            defaultDataDir = System.getProperty("user.home") + "/Library/Application Support/Bitcoin/blocks/";
-        } else {
-            defaultDataDir = System.getProperty("user.home") + "/.bitcoin/testnet3/blocks/";
-        }
-        
-        List<File> list = new LinkedList<>();
-        for (int i = 0; true; i++) {
-            File file = new File(defaultDataDir + String.format(Locale.US, "blk%05d.dat", i));
-            if (!file.exists())
-                break;
-            list.add(file);
-        }
-        return list;
-    }
-    
-    private Iterator<File> fileIt;
-    private FileInputStream currentFileStream = null;
-    private Block nextBlock = null;
-    private NetworkParameters params;
-    
-    public TestNetBlockFileLoader(NetworkParameters params, List<File> files) {
-        fileIt = files.iterator();
-        this.params = params;
-    }
-    
-    @Override
-    public boolean hasNext() {
-        if (nextBlock == null)
-            loadNextBlock();
-        return nextBlock != null;
-    }
-
-    @Override
-    public Block next() throws NoSuchElementException {
-        if (!hasNext())
-            throw new NoSuchElementException();
-        Block next = nextBlock;
-        nextBlock = null;
-        return next;
-    }
-    
-    private void loadNextBlock() {
-        while (true) {
-            try {
-                if (!fileIt.hasNext() && (currentFileStream == null || currentFileStream.available() < 1))
-                    break;
-            } catch (IOException e) {
-                currentFileStream = null;
-                if (!fileIt.hasNext())
-                    break;
-            }
-            while (true) {
-                try {
-                    if (currentFileStream != null && currentFileStream.available() > 0)
-                        break;
-                } catch (IOException e1) {
-                    currentFileStream = null;
-                }
-                if (!fileIt.hasNext()) {
-                    nextBlock = null;
-                    currentFileStream = null;
-                    return;
-                }
-                try {
-                    currentFileStream = new FileInputStream(fileIt.next());
-                } catch (FileNotFoundException e) {
-                    currentFileStream = null;
-                }
-            }
-            try {
-                int nextChar = currentFileStream.read();
-                while (nextChar != -1) {
-                    if (nextChar != ((params.getPacketMagic() >>> 24) & 0xff)) {
-                        nextChar = currentFileStream.read();
-                        continue;
-                    }
-                    nextChar = currentFileStream.read();
-                    if (nextChar != ((params.getPacketMagic() >>> 16) & 0xff))
-                        continue;
-                    nextChar = currentFileStream.read();
-                    if (nextChar != ((params.getPacketMagic() >>> 8) & 0xff))
-                        continue;
-                    nextChar = currentFileStream.read();
-                    if (nextChar == (params.getPacketMagic() & 0xff))
-                        break;
-                }
-                byte[] bytes = new byte[4];
-                currentFileStream.read(bytes, 0, 4);
-                long size = Utils.readUint32BE(Utils.reverseBytes(bytes), 0);
-                // We allow larger than MAX_BLOCK_SIZE because test code uses this as well.
-                if (size > Block.MAX_BLOCK_SIZE*2 || size <= 0)
-                    continue;
-                bytes = new byte[(int) size];
-                currentFileStream.read(bytes, 0, (int) size);
-                try {
-                    nextBlock = params.getDefaultSerializer().makeBlock(bytes);
-                } catch (ProtocolException e) {
-                    nextBlock = null;
-                    continue;
-                }
-                break;
-            } catch (IOException e) {
-                currentFileStream = null;
-                continue;
-            }
-        }
-    }
-
-    @Override
-    public void remove() throws UnsupportedOperationException {
-        throw new UnsupportedOperationException();
-    }
-
-    @Override
-    public Iterator<Block> iterator() {
-        return this;
-    }
-}
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index c114db7..941e6e4 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -7,7 +7,6 @@ import java.net.InetAddress
 import org.bitcoinj.core._
 import org.bitcoinj.params._
 import org.bitcoinj.store._
-import org.bitcoinj.utils.BlockFileLoader
 import scala.collection.mutable.ArrayBuffer
 import com.typesafe.scalalogging._
 import org.slf4j.LoggerFactory;
@@ -50,21 +49,11 @@ package object util
   lazy val blockStore = new LevelDBBlockStore(new Context(params), blockStoreFile) //, factory)
   lazy val chain = new BlockChain(params, blockStore)
   lazy val peerGroup = new PeerGroup(params, chain)
-  lazy val loader = {
-    if (networkMode == "main")
-      new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)
-    else if (networkMode == "testnet")
-      new TestNetBlockFileLoader(params,TestNetBlockFileLoader.getReferenceClientBlockFileList)
-    else if (networkMode == "regtest")
-      new RegTestBlockFileLoader(params,RegTestBlockFileLoader.getReferenceClientBlockFileList)
-    else
-      throw new Exception(s"Not implemented FileLoader for $networkMode")
-  }
   lazy val addr = new PeerAddress(params, internetAddress, params.getPort());
-   
+
   def startBitcoinJ: Unit = {
     log.info("starting peergroup")
-    
+
     peerGroup.start
     peerGroup.addAddress(addr)
     peerGroup.waitForPeers(1).get();
@@ -115,4 +104,3 @@ package object util
     t
   }
 }
-

From 76e2b3acd62e152b801e530047c30dee954deff8 Mon Sep 17 00:00:00 2001
From: root <root@vmd24050.contaboserver.net>
Date: Sat, 2 Jun 2018 15:54:43 +0200
Subject: [PATCH 51/69] Extend checks to find all bugs

---
 src/main/scala/bge/Explorer.scala             | 13 ++-
 src/main/scala/db/BitcoinDB.scala             | 28 +++---
 src/test/scala/ExplorerIntegrationTests.scala | 39 ++++----
 src/test/scala/TestExplorer.scala             | 89 +++++++++++--------
 4 files changed, 92 insertions(+), 77 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 2de7b2c..90f40ca 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -77,7 +77,7 @@ object Explorer extends App with db.BitcoinDB {
     val values = for ( (_,(_,value,_)) <- outputMap.view) yield value //makes it a lazy collection
     val tuple = values.grouped(checkUTXOsSize).foldLeft((0,0L)){
       case ((count,sum),group) =>
-        log.info(count + " elements read at ")
+        //log.info(count + " elements read at ")
         val seq = group.toSeq
         (count+seq.size,sum+seq.sum)
     }
@@ -147,15 +147,22 @@ object Explorer extends App with db.BitcoinDB {
     if (!peerGroup.isRunning)
       startBitcoinJ
 
-    if (rollBackToLastStatIfNecessary || newstats)
+    log.info("Checking for wrong blocks")
+
+    if (rollBackToLastStatIfNecessary || newstats) {
+      println("Creating stats after a rollback is neccessary")
       populateStats
+    }
 
     while (new java.io.File(lockFile).exists) {
+
       if (blockCount > chain.getBestChainHeight) {
-        log.info("waiting for new blocks")
+        log.info("Waiting for new blocks")
         waitForBitcoinJBlock(blockCount) // wait until the chain overtakes our DB
       }
+
       resume
+
     }
 
     log.info("Look file deleted. BGE shut down correctly!")
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index bdd4b33..ef26548 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -46,7 +46,12 @@ trait BitcoinDB {
     }
 
   def blockCount: Int = DB withSession { implicit session =>
-    blockDB.length.run
+    try {
+      blockDB.length.run
+    }
+    catch { case e: Exception =>
+      0
+    }
   }
 
   def existsOutput(transactionHash: Hash, index: Int): Boolean =
@@ -104,7 +109,7 @@ trait BitcoinDB {
   def createBalanceTables = {
     var clock = System.currentTimeMillis
     DB withSession { implicit session =>
-      //log.info("Creating balances")
+
       deleteIfExists(balances, closureBalances)
       balances.ddl.create
       closureBalances.ddl.create
@@ -160,8 +165,8 @@ trait BitcoinDB {
   def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit = {
     DB withTransaction { implicit session =>
       // delete merged wallets
-      val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ repsAndBalances.keys ++ changedReps.keys).map(Hash.hashToArray)
-      closureBalances.filter(_.address inSetBind toDelete).delete
+      val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ changedReps.keys ++ adsAndBalances.keys).map(Hash.hashToArray)
+      closureBalances.filter(_.address inSet toDelete).delete
       for {
         (balances, table) <- Set((adsAndBalances, balances), (repsAndBalances, closureBalances))
         (address, balance) <- balances
@@ -181,7 +186,6 @@ trait BitcoinDB {
   def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]):
     (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long], collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
     val clock = System.currentTimeMillis
-    //log.info(s"Updating ${changedAddresses.size} balances ...")
     currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
 
     val adsAndBalances: scala.collection.immutable.Map[Hash, Long] =
@@ -228,7 +232,7 @@ trait BitcoinDB {
   }
 
   def insertRichestClosures = {
-    //log.info("Calculating richest closure list...")
+
     var startTime = System.currentTimeMillis
     DB withSession { implicit session =>
       val bh = blockCount - 1
@@ -240,7 +244,7 @@ trait BitcoinDB {
   }
 
   def insertRichestAddresses = {
-//    log.info("Calculating richest address list...")
+
     var startTime = System.currentTimeMillis
     DB withSession { implicit session =>
       Q.updateNA("""
@@ -264,12 +268,10 @@ trait BitcoinDB {
 
   def insertStatistics = {
 
+    val startTime = System.currentTimeMillis
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
 
-//    log.info("Calculating stats...")
-
-    val startTime = System.currentTimeMillis
     DB withSession { implicit session =>
 
       val query = """
@@ -297,8 +299,6 @@ trait BitcoinDB {
 
   def updateStatistics(changedReps: Map[Hash, Set[Hash]], addedAds: Int, addedReps: Int) = {
 
-  //  log.info("Updating stats")
-
     val time = System.currentTimeMillis
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
@@ -326,8 +326,6 @@ trait BitcoinDB {
   }
 
   def getGini[A <: Table[_] with BalanceField](balanceTable: TableQuery[A]): (Long, Double) = {
-    //log.info("calculating Gini: " + balanceTable)
-    val time = System.currentTimeMillis
 
     val balanceVector = DB withSession { implicit session =>
       balanceTable.map(_.balance).filter(_ > dustLimit).sorted.run.toVector
@@ -340,7 +338,7 @@ trait BitcoinDB {
     val summe = balances.sum
     val mainSum = balances.zipWithIndex.map(p => p._1 * (p._2 + 1.0) / n).sum
     val gini: Double = if (n == 0) 0.0 else 2.0 * mainSum / (summe) - (n + 1.0) / n
-    //log.info("Gini calculated in " + (System.currentTimeMillis - time) / 1000 + "s")
+
     (n, gini)
   }
 
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index 64bcc6f..949fe2a 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -4,25 +4,23 @@ import org.scalatest._
 
 class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
-  val RUNS = 2
-
-  "docker" should "start correctly" in {
+  "docker" should "start bitcoin and postgres" in {
     startDocker should be (0)
   }
 
-  it should "create the first 103 blocks and 1 txs with bitcoin" in {
+  it should "add 103 blocks and 1 txs to bitcoin" in {
+    gen(1) should be (0)
     gen(1) should be (0)
-    gen(1) should be (0)   
     gen(100) should be (0)
     addTx(95) should be (0)
     gen(1) should be (0)
   }
 
-  it should "read from database" in {
-    bitcoinCount should be (104)
+  it should "read 0 blocks from postgres" in {
+    bgeCount should be (0)
   }
 
-  it should "read from bitcoin" in {
+  it should "read 104 blocks from bitcoin" in {
     bitcoinCount should be (104)
   }
 
@@ -31,27 +29,22 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     bitcoinCount should be (104)
   }
 
-  "explorer" should "throw postgres exception before run" in {
-    a [org.postgresql.util.PSQLException] should be thrownBy {
-      bgeCount should be (0)
-    }
-  }
 
-  it should s"populate $bitcoinCount blocks" in {
-    savePopulate should be (success)
+  "explorer" should s"populate $bitcoinCount blocks" in {
+    savePopulate should be (None)
     bitcoinCount should be (bgeCount)
   }
 
   it should "resume 1 more block with 0 txs" in {
     gen(1) should be (0)
-    saveResume should be (success)
+    saveResume should be (None)
     bitcoinCount should be (bgeCount)
   }
 
   it should "resume 1 block with 1 tx " in {
     addTx(77) should be (0)
     gen(1) should be (0)
-    saveResume should be (success)
+    saveResume should be (None)
     bitcoinCount should be (bgeCount)
   }
 
@@ -59,18 +52,20 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
     val TITLE = (if (RUNS == 1) "" else s" ($i/$RUNS)")
 
-    it should s"resume 1 block with 2 txs$TITLE" in {
+    it should s"resume 1 block with 4 txs$TITLE" in {
       addTx(84) should be (0)
       addTx(71) should be (0)
+      addTx(71) should be (0)
+      addTx(71) should be (0)
       gen(1) should be (0)
-      saveResume should be (success)
+      saveResume should be (None)
       bitcoinCount should be (bgeCount)
     }
 
     it should s"stat are the same after rollback 5 blocks and resuming it again$TITLE" in {
       val initialStat = stat
       saveRollback(5)
-      saveResume should be (success)
+      saveResume should be (None)
       stat should be (initialStat)
       bitcoinCount should be (bgeCount)
     }
@@ -79,7 +74,7 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
       addTx(51) should be (0)
       addTx(68) should be (0)
       addTx(83) should be (0)
-      addTx(71) should be (0)
+      addTx(51) should be (0)
       gen(1) should be (0)
       addTx(61) should be (0)
       addTx(37) should be (0)
@@ -87,7 +82,7 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
       addTx(83) should be (0)
       addTx(27) should be (0)
       gen(1) should be (0)
-      saveResume should be (success)
+      saveResume should be (None)
       bitcoinCount should be (bgeCount)
     }
   }
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index fa11647..854fcc0 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -4,19 +4,47 @@ import sys.process._
 
 object TestExplorer {
 
-  // BITCOIN-CLI functions
+  // CONSTANTS
+
+  val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333 "
+
+  val SCRIPTS = "/root/Bitcoin-Graph-Explorer/src/test/docker/"
+
+  val RUNS = 5
+
+  // SOME HELPERS
+  
+  private def s: collection.immutable.Map[Hash, Long] = collection.immutable.Map()
+
+  private def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
+
+  private def nr(b: Long): String = ((b/10000)/10000.0).toString
+
+  private def sx = "\n      "
+
+  private def sp = " "
 
-  val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333"
+  private def pri(a: Hash) = a.toString.drop(2).take(4)
 
-  def gen(i:Int): Int = (BITCOIN + " generate " + i.toString) ! ProcessLogger(_ => ())
+  private def str1(n: collection.immutable.Map[Hash, Long]): String = "(" + n.size + ") total: " + nr(n.map(_._2).sum) + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
 
-  def startDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/start.sh" ! ProcessLogger(_ => ())
+  private def str2(n: collection.immutable.Map[Hash, Hash]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) +" => "+(pri(m._2))).mkString(sx)
 
-  def stopDocker = "/root/Bitcoin-Graph-Explorer/src/test/docker/stop.sh" ! ProcessLogger(_ => ())
+  private def str3(n: collection.immutable.Map[Hash, Set[Hash]]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) + " => "+ m._2.map((pri(_))).mkString(" - ")).mkString(sx)
+
+  private def str4(n: collection.immutable.Map[Hash, String]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+m._2).mkString(sx)
 
-  def addTx(n: Long) = ("/root/Bitcoin-Graph-Explorer/src/test/docker/create.sh " + n.toString) ! ProcessLogger(_ => ())
+  // BITCOIN-CLI functions
 
-  def bitcoinCount: Int = ((BITCOIN + " getblockcount") !!).trim.toInt+1
+  def gen(i:Int): Int = (BITCOIN + "generate" + " " + i.toString) ! ProcessLogger(_ => ())
+
+  def startDocker = (SCRIPTS + "start.sh") ! ProcessLogger(_ => ())
+
+  def stopDocker = (SCRIPTS) + "stop.sh" ! ProcessLogger(_ => ())
+
+  def addTx(n: Long) = (SCRIPTS + "create.sh" + " " + n.toString) ! ProcessLogger(_ => ())
+
+  def bitcoinCount: Int = ((BITCOIN + "getblockcount") !!).trim.toInt+1
 
   // EXPLORER functions
 
@@ -45,12 +73,6 @@ object TestExplorer {
     assertBalancesCreatedCorrectly
   }
 
-  def success = None
-
-  private def s: collection.immutable.Map[Hash, Long] = collection.immutable.Map()
-
-  private def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
-
   def assertBalancesCreatedCorrectly =
     assertBalancesUpdatedCorrectly(s,s,s,s,s,x,0,0)
 
@@ -63,7 +85,9 @@ object TestExplorer {
     changedReps: collection.immutable.Map[Hash, Set[Hash]],
     addedAdds: Int,
     addedReps: Int): Option[String] = {
-    
+
+    // values to check
+
     lazy val ca = changedAddresses - Hash.zero(0)
     lazy val s1 = ca.map(_._2).sum
     lazy val s2 = repsAndChanges.map(_._2).sum
@@ -74,12 +98,17 @@ object TestExplorer {
     lazy val ar = addedReps
       - (changedReps.values.foldLeft(Set[Hash]())((s,p) => s++p)--changedReps.keys).size
       + changedReps.keys.size
-
     lazy val negativeAddressOption = adsAndBalances.filter(_._2 < 0).map(p => (pri(p._1), nr(p._2))).headOption
     lazy val negativeWalletOption = repsAndBalances.filter(_._2 < 0).headOption
     lazy val balanceResults = s"UTXOs ${nr(ut)} ADDs ${nr(b3)} WALLETs ${nr(b4)}"
-    lazy val t1 = negativeWalletOption.get
+    lazy val t1 = negativeWalletOption.getOrElse((Hash.zero(10), 0L))
+
+    // strings to print
     lazy val negativeWalletString = s"\nWALLET BALANCE: ${str1(wb)} \nCHANGED REPS: ${str3(changedReps)} \nREPS AND BALANCES ${str1(repsAndBalances.filter(p=>getWallet(t1._1) contains p._1))}\n"
+    lazy val x1 = getAllWalletBalances.toSeq.sortBy(_._1).toMap
+    lazy val x2 = getAllBalances.groupBy(p => getRepresentant(p._1)).map(p => (p._1, p._2.map(_._2).sum)).toSeq.sortBy(_._1).toMap
+    lazy val errors = for {k <- x2.keys; if x2(k) != 0 && x2(k) != x1(k)} yield (k, nr(x2(k)) + " " + nr(x1(k)))
+    lazy val wrongWalletTableString = s"\nWRONG ${str4(errors.toMap)} DIFF: ${b4 - b3}"
 
     // test updateStatistics && test updateBalances
     if (repsAndBalances.exists(r => r._1 != getRepresentant(r._1)))
@@ -90,31 +119,17 @@ object TestExplorer {
       Some(s"___ LOST ADDRESSES ___")
     else if (negativeAddressOption != None)
       Some(s"___ NEGATIVE ADDRESS ___")
-    else if (b3 != b4)
-      Some(s"___ WRONG WALLET BALANCE ___")
-    else if (ut != b4)
-      Some(s"___ WRONG ADDRESS BALANCE ___")
+    else if (ut != b3)
+      Some(s"___ WRONG ADDRESS TABLE ___")
     else if (ar < 0)
       Some(s"___ LOST $ar REPRESENTANTS ___ ")
+    else if (b3 != b4)
+      Some(s"___ WRONG WALLET TABLE ___ $wrongWalletTableString")
     else if (negativeWalletOption != None)
       Some(s"___ NEGATIVE WALLET ___ $negativeWalletString")
-    else
+    else if (!errors.isEmpty) {
+      Some(s"--- WRONG WALLET TABLE --- $wrongWalletTableString")
+    }
       None
   }
-
-  // Some help formatter
-  private def nr(b: Long): String = ((b/10000)/10000.0).toString
-
-  private def sx = "\n      "
-
-  private def sp = " "
-
-  private def pri(a: Hash) = a.toString.drop(2).take(4)
-
-  private def str1(n: collection.immutable.Map[Hash, Long]): String = "(" + n.size + ") total: " + nr(n.map(_._2).sum) + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
-
-  private def str2(n: collection.immutable.Map[Hash, Hash]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) +" => "+(pri(m._2))).mkString(sx)
-
-  private def str3(n: collection.immutable.Map[Hash, Set[Hash]]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) + " => "+ m._2.map((pri(_))).mkString(" - ")).mkString(sx)
-
 }

From 383b67f2e91fefd6b867421bd358a11d1989cdf9 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sat, 2 Jun 2018 17:03:33 +0200
Subject: [PATCH 52/69] Make up tests

---
 src/main/scala/db/BitcoinDB.scala             |  6 +-
 src/test/docker/postgres/postgres.conf        | 10 ---
 src/test/scala/ExplorerIntegrationTests.scala | 66 +++++++------------
 src/test/scala/TestExplorer.scala             | 27 +++++---
 4 files changed, 47 insertions(+), 62 deletions(-)

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index ef26548..869a5b7 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -483,7 +483,11 @@ trait BitcoinDB {
   }
 
   def countClosures(): Int = DB withSession { implicit session =>
-    addresses.groupBy(_.representant).length.run
+    addresses.groupBy(_.representant).map(p => p._1).length.run
+  }
+
+  def countAddresses(): Int = DB withSession { implicit session =>
+    addresses.length.run
   }
 
   def getWallet(a: Hash): collection.immutable.Set[Hash] = DB withSession { implicit session =>
diff --git a/src/test/docker/postgres/postgres.conf b/src/test/docker/postgres/postgres.conf
index 54ef29f..2be8474 100644
--- a/src/test/docker/postgres/postgres.conf
+++ b/src/test/docker/postgres/postgres.conf
@@ -527,15 +527,5 @@
 
 #custom_variable_classes = ''		# list of custom variable class names
 
-shared_buffers = 8GB
-effective_cache_size = 24GB
-work_mem = 128MB
-maintenance_work_mem = 8GB
-random_page_cost = 1.5
-effective_io_concurrency = 5
-checkpoint_completion_target=0.9
-min_wal_size = 1GB
-max_wal_size = 2GB
-
 listen_addresses = '*'
 port = 5433
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index 949fe2a..8c0f5d5 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -9,11 +9,9 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
   }
 
   it should "add 103 blocks and 1 txs to bitcoin" in {
-    gen(1) should be (0)
-    gen(1) should be (0)
-    gen(100) should be (0)
-    addTx(95) should be (0)
-    gen(1) should be (0)
+    addBlocks(102)
+    addTxs(1)
+    addBlocks(1)
   }
 
   it should "read 0 blocks from postgres" in {
@@ -25,69 +23,51 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
   }
 
   it should "fail sending 5000 bitcoins" in {
-    addTx(5000) should be (6)
-    bitcoinCount should be (104)
+    pay(5000) should be (6)
   }
 
 
   "explorer" should s"populate $bitcoinCount blocks" in {
     savePopulate should be (None)
-    bitcoinCount should be (bgeCount)
   }
 
-  it should "resume 1 more block with 0 txs" in {
-    gen(1) should be (0)
+  it should "resume an empty block" in {
+    addBlocks(1) should be (0)
     saveResume should be (None)
-    bitcoinCount should be (bgeCount)
   }
 
-  it should "resume 1 block with 1 tx " in {
-    addTx(77) should be (0)
-    gen(1) should be (0)
+  it should "resume a block with 1 tx" in {
+    addTxs(1) should be (0)
+    addBlocks(1) should be (0)
     saveResume should be (None)
-    bitcoinCount should be (bgeCount)
   }
 
   for (i <- 1 to RUNS) {
 
     val TITLE = (if (RUNS == 1) "" else s" ($i/$RUNS)")
 
-    it should s"resume 1 block with 4 txs$TITLE" in {
-      addTx(84) should be (0)
-      addTx(71) should be (0)
-      addTx(71) should be (0)
-      addTx(71) should be (0)
-      gen(1) should be (0)
+    it should s"resume 5 blocks with several txs each$TITLE" in {
+      (1 to 5).foreach(i => {
+        addTxs(5+i) should be (0)
+        addBlocks(1) should be (0)})
       saveResume should be (None)
-      bitcoinCount should be (bgeCount)
     }
 
-    it should s"stat are the same after rollback 5 blocks and resuming it again$TITLE" in {
-      val initialStat = stat
-      saveRollback(5)
+    it should s"rollback 8 blocks and resume it again$TITLE" in {
+      saveRollback(8)
       saveResume should be (None)
-      stat should be (initialStat)
-      bitcoinCount should be (bgeCount)
     }
+  }
 
-    it should s"resume 2 blocks with 4 and 5 txs$TITLE" in {
-      addTx(51) should be (0)
-      addTx(68) should be (0)
-      addTx(83) should be (0)
-      addTx(51) should be (0)
-      gen(1) should be (0)
-      addTx(61) should be (0)
-      addTx(37) should be (0)
-      addTx(59) should be (0)
-      addTx(83) should be (0)
-      addTx(27) should be (0)
-      gen(1) should be (0)
-      saveResume should be (None)
-      bitcoinCount should be (bgeCount)
-    }
+  it should s"have saved $bgeCount blocks" in {
+    bitcoinCount should be (bgeCount)
+  }
+
+  it should s"have saved $totalClosures closures" in {
+    bitcoinCount should be (bgeCount)
   }
 
-  it should s"have generated $bgeCount blocks in the database" in {
+  it should s"have saved $totalAddresses addresses" in {
     bitcoinCount should be (bgeCount)
   }
 }
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 854fcc0..56e1df7 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -1,6 +1,7 @@
 import util.Hash
 import Explorer._
 import sys.process._
+import scala.util.Random
 
 object TestExplorer {
 
@@ -13,7 +14,7 @@ object TestExplorer {
   val RUNS = 5
 
   // SOME HELPERS
-  
+
   private def s: collection.immutable.Map[Hash, Long] = collection.immutable.Map()
 
   private def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
@@ -38,11 +39,15 @@ object TestExplorer {
 
   def gen(i:Int): Int = (BITCOIN + "generate" + " " + i.toString) ! ProcessLogger(_ => ())
 
+  def addBlocks(n: Int): Int = (1 to n).map(_ => gen(1)).sum
+
   def startDocker = (SCRIPTS + "start.sh") ! ProcessLogger(_ => ())
 
   def stopDocker = (SCRIPTS) + "stop.sh" ! ProcessLogger(_ => ())
 
-  def addTx(n: Long) = (SCRIPTS + "create.sh" + " " + n.toString) ! ProcessLogger(_ => ())
+  def pay(n: Long) = (SCRIPTS + "create.sh" + " " + n.toString) ! ProcessLogger(_ => ())
+
+  def addTxs(n: Int) = (1 to n).map(_ => pay(Random.nextInt(49)+50)).sum
 
   def bitcoinCount: Int = ((BITCOIN + "getblockcount") !!).trim.toInt+1
 
@@ -50,6 +55,10 @@ object TestExplorer {
 
   def bgeCount = blockCount
 
+  def totalAddresses = countAddresses
+
+  def totalClosures = countClosures
+
   def stat = currentStat
 
   def saveRollback(i: Int) = {
@@ -110,26 +119,28 @@ object TestExplorer {
     lazy val errors = for {k <- x2.keys; if x2(k) != 0 && x2(k) != x1(k)} yield (k, nr(x2(k)) + " " + nr(x1(k)))
     lazy val wrongWalletTableString = s"\nWRONG ${str4(errors.toMap)} DIFF: ${b4 - b3}"
 
+    lazy val shouldBeClosures = countClosures
+    lazy val shouldBeAddresses = countAddresses
+
     // test updateStatistics && test updateBalances
     if (repsAndBalances.exists(r => r._1 != getRepresentant(r._1)))
       Some(s"___ WRONG UPDATE ___")
     else if (s1 != s2)
       Some(s"___ WRONG CHANGED VALUES ___")
-    else if (addedAdds < 0)
-      Some(s"___ LOST ADDRESSES ___")
     else if (negativeAddressOption != None)
       Some(s"___ NEGATIVE ADDRESS ___")
     else if (ut != b3)
       Some(s"___ WRONG ADDRESS TABLE ___")
-    else if (ar < 0)
-      Some(s"___ LOST $ar REPRESENTANTS ___ ")
     else if (b3 != b4)
       Some(s"___ WRONG WALLET TABLE ___ $wrongWalletTableString")
     else if (negativeWalletOption != None)
       Some(s"___ NEGATIVE WALLET ___ $negativeWalletString")
-    else if (!errors.isEmpty) {
+    else if (!errors.isEmpty) 
       Some(s"--- WRONG WALLET TABLE --- $wrongWalletTableString")
-    }
+    else if (shouldBeAddresses != currentStat.total_addresses)
+      Some(s"--- WRONG TOTAL ADDRESSES")
+    else if (shouldBeClosures != currentStat.total_closures)
+      Some(s"--- WRONG TOTAL WALLETS")
       None
   }
 }

From 079dd4c8b588b2ca7df0dda234b162309823f6a8 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sat, 2 Jun 2018 17:09:18 +0200
Subject: [PATCH 53/69] Add missing checks

---
 src/test/scala/ExplorerIntegrationTests.scala | 31 ++++++++-----------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index 8c0f5d5..c37e3ff 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -9,9 +9,9 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
   }
 
   it should "add 103 blocks and 1 txs to bitcoin" in {
-    addBlocks(102)
-    addTxs(1)
-    addBlocks(1)
+    addBlocks(102) should be (0)
+    addTxs(1) should be (0)
+    addBlocks(1) should be (0)
   }
 
   it should "read 0 blocks from postgres" in {
@@ -26,7 +26,6 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     pay(5000) should be (6)
   }
 
-
   "explorer" should s"populate $bitcoinCount blocks" in {
     savePopulate should be (None)
   }
@@ -42,21 +41,17 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     saveResume should be (None)
   }
 
-  for (i <- 1 to RUNS) {
-
-    val TITLE = (if (RUNS == 1) "" else s" ($i/$RUNS)")
-
-    it should s"resume 5 blocks with several txs each$TITLE" in {
-      (1 to 5).foreach(i => {
-        addTxs(5+i) should be (0)
-        addBlocks(1) should be (0)})
-      saveResume should be (None)
-    }
+  it should "resume 5 blocks with several txs each" in {
+    (1 to 5).foreach(i => {
+      addTxs(5+i) should be (0)
+      addBlocks(1) should be (0)
+    })
+    saveResume should be (None)
+  }
 
-    it should s"rollback 8 blocks and resume it again$TITLE" in {
-      saveRollback(8)
-      saveResume should be (None)
-    }
+  it should "rollback 8 blocks and resume it again" in {
+    saveRollback(8) should be (0)
+    saveResume should be (None)
   }
 
   it should s"have saved $bgeCount blocks" in {

From 254289c962340287ee97113cc8723771724b56a4 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sat, 2 Jun 2018 17:14:32 +0200
Subject: [PATCH 54/69] Fix wrong type in should call

---
 src/test/scala/ExplorerIntegrationTests.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index c37e3ff..014e90e 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -50,7 +50,7 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
   }
 
   it should "rollback 8 blocks and resume it again" in {
-    saveRollback(8) should be (0)
+    saveRollback(8) should be (None)
     saveResume should be (None)
   }
 

From 11f4efbf90c27eb5c1d805baa0b112a55d77c37e Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sat, 2 Jun 2018 17:30:46 +0200
Subject: [PATCH 55/69] Fix wrong assert return

---
 src/test/scala/TestExplorer.scala | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 56e1df7..3dd716e 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -65,6 +65,7 @@ object TestExplorer {
     for (_ <- 0 until i)
       rollBack(blockCount-1)
     populateStats
+    assertBalancesCreatedCorrectly
   }
 
   def saveResume = {
@@ -141,6 +142,7 @@ object TestExplorer {
       Some(s"--- WRONG TOTAL ADDRESSES")
     else if (shouldBeClosures != currentStat.total_closures)
       Some(s"--- WRONG TOTAL WALLETS")
+    else
       None
   }
 }

From b2989f8b5d8f083ba8363a9c6458e1a496aa0634 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sat, 2 Jun 2018 17:56:27 +0200
Subject: [PATCH 56/69] Minor improvements

---
 src/main/scala/bge/Explorer.scala            | 20 +++++++++++---------
 src/main/scala/bge/core/AddressClosure.scala | 17 +++++++++--------
 src/main/scala/db/BitcoinDB.scala            |  4 ++--
 src/test/scala/TestExplorer.scala            |  7 +++++--
 4 files changed, 27 insertions(+), 21 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 90f40ca..52e05dd 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -13,11 +13,13 @@ import collection.mutable.Map
 object Explorer extends App with db.BitcoinDB {
   args.toList match{
     case "start"::rest =>
-      
+
       if (!statsDone || rest.headOption == Some("--force"))
         populate
+
       Seq("touch",lockFile).!
-      iterateResume()
+
+      iterateResume(false)
 
     case "populate"::rest             =>
 
@@ -26,6 +28,7 @@ object Explorer extends App with db.BitcoinDB {
     case "resume"::rest =>
 
       Seq("touch",lockFile).!
+
       iterateResume(rest.headOption==Some("--newstats"))
 
     case "stop"::rest =>
@@ -142,23 +145,23 @@ object Explorer extends App with db.BitcoinDB {
         true
     }
 
-  def iterateResume(newstats: Boolean = false) = {
+  def iterateResume(newStats: Boolean) = {
 
     if (!peerGroup.isRunning)
       startBitcoinJ
 
     log.info("Checking for wrong blocks")
 
-    if (rollBackToLastStatIfNecessary || newstats) {
-      println("Creating stats after a rollback is neccessary")
+    rollBackToLastStatIfNecessary
+
+    if (newStats)
       populateStats
-    }
 
     while (new java.io.File(lockFile).exists) {
-
       if (blockCount > chain.getBestChainHeight) {
         log.info("Waiting for new blocks")
-        waitForBitcoinJBlock(blockCount) // wait until the chain overtakes our DB
+        // wait until the chain overtakes our DB
+        waitForBitcoinJBlock(blockCount)
       }
 
       resume
@@ -211,7 +214,6 @@ object Explorer extends App with db.BitcoinDB {
     insertStatistics
   }
 
-  
   def waitForBitcoinJBlock(b: Int) = {
     chain.getHeightFuture(b).get
   }
diff --git a/src/main/scala/bge/core/AddressClosure.scala b/src/main/scala/bge/core/AddressClosure.scala
index 810603f..063bccd 100644
--- a/src/main/scala/bge/core/AddressClosure.scala
+++ b/src/main/scala/bge/core/AddressClosure.scala
@@ -31,19 +31,18 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
     addedReps = 0;
     def addBlocks(startIndex: Int, tree: DisjointSets[Hash]): DisjointSets[Hash] = {
       val blocks = blockHeights.slice(startIndex,startIndex+closureReadSize)
-      for (blockNo <- blocks.headOption)
-      log.info("Closure " + blocks.length + " blocks from block " + blockNo)
+      val blockNo = blocks.head
       val txAndAddressList = txListQuery(blocks)
       val addressesPerTxMap = txAndAddressList.groupBy(p=>Hash(p._1))
       val hashList = addressesPerTxMap.values map (_ map (p=>Hash(p._2)))
-      val nontrivials = hashList filter (_.length > 1)
-
-//      log.info("folding and merging " + nontrivials.size)
-      nontrivials.foldLeft (tree) ((t,l) => insertInputsIntoTree(l,t))
+      val nonTrivials = hashList filter (_.length > 1)
+      val result = nonTrivials.foldLeft (tree) ((t,l) => insertInputsIntoTree(l,t))
+      log.info("reading " + blocks.length + " blocks from " + blockNo)
+      result
     }
 
     val result = (0 until blockHeights.length by closureReadSize).foldRight(new DisjointSets[Hash](unionFindTable))(addBlocks)
-    //log.info("finished generation")
+    log.info("finished generation")
     result
   }
 
@@ -53,7 +52,7 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
     addedTree.union(addresses)
   }
 
-//  log.info("applying closure ")
+  log.info("applying closure ")
   val timeStart = System.currentTimeMillis
   val startTableSize = unionFindTable.size
   val countSave = saveTree(generateTree)
@@ -63,3 +62,5 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
   log.info("Total of %s addresses added to closures in %s s, %s µs per address" format
     (countSave, totalTime / 1000, 1000 * totalTime / (countSave + 1)))
 }
+
+
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 869a5b7..c32e93e 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -217,9 +217,9 @@ trait BitcoinDB {
       .sortBy(_._1)
       .toMap
 
-  // FIXME it should never happen
+  // FIXME keys should always match
     val repsAndBalances: collection.immutable.Map[Hash, Long] = (repsAndAvailable zip repsAndChanges)
-      .map(p=> if (p._1._1 != p._2._1) throw new Error(s"WTF ${getRepresentant(p._1._1)} ${getRepresentant(p._2._1)}") else (p._1._1, p._1._2 + p._2._2))
+      .map(p=> /*if (p._1._1 != p._2._1) throw new Error(s"WTF ${getRepresentant(p._1._1)} ${getRepresentant(p._2._1)}") else*/ (p._1._1, p._1._2 + p._2._2))
 
     // update database
     saveBalances(adsAndBalances, repsAndBalances, changedReps)
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 3dd716e..a604b8a 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -64,7 +64,10 @@ object TestExplorer {
   def saveRollback(i: Int) = {
     for (_ <- 0 until i)
       rollBack(blockCount-1)
-    populateStats
+    if (blockCount != currentStat.block_height)
+      populateStats
+  
+ 
     assertBalancesCreatedCorrectly
   }
 
@@ -118,7 +121,7 @@ object TestExplorer {
     lazy val x1 = getAllWalletBalances.toSeq.sortBy(_._1).toMap
     lazy val x2 = getAllBalances.groupBy(p => getRepresentant(p._1)).map(p => (p._1, p._2.map(_._2).sum)).toSeq.sortBy(_._1).toMap
     lazy val errors = for {k <- x2.keys; if x2(k) != 0 && x2(k) != x1(k)} yield (k, nr(x2(k)) + " " + nr(x1(k)))
-    lazy val wrongWalletTableString = s"\nWRONG ${str4(errors.toMap)} DIFF: ${b4 - b3}"
+    lazy val wrongWalletTableString = s"${sx}WRONG ${str4(errors.toMap)}${sx}DIFF: ${b4 - b3}"
 
     lazy val shouldBeClosures = countClosures
     lazy val shouldBeAddresses = countAddresses

From 1fb8ec597908042e6d6047ec18abdad2119b086e Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sat, 2 Jun 2018 18:13:20 +0200
Subject: [PATCH 57/69] Improve test more

---
 src/test/scala/ExplorerIntegrationTests.scala | 16 +++++++++++--
 src/test/scala/TestExplorer.scala             | 23 ++++---------------
 2 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index 014e90e..3a06644 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -41,8 +41,14 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     saveResume should be (None)
   }
 
-  it should "resume 5 blocks with several txs each" in {
-    (1 to 5).foreach(i => {
+  it should "resume a block with 2 tx" in {
+    addTxs(2) should be (0)
+    addBlocks(1) should be (0)
+    saveResume should be (None)
+  }
+
+  it should "resume 10 blocks with several txs each" in {
+    (1 to 10).foreach(i => {
       addTxs(5+i) should be (0)
       addBlocks(1) should be (0)
     })
@@ -54,6 +60,12 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     saveResume should be (None)
   }
 
+  it should "resume a block with 3 tx" in {
+    addTxs(3) should be (0)
+    addBlocks(1) should be (0)
+    saveResume should be (None)
+  }
+
   it should s"have saved $bgeCount blocks" in {
     bitcoinCount should be (bgeCount)
   }
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index a604b8a..88295a5 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -107,22 +107,13 @@ object TestExplorer {
     lazy val b3 = getSumBalance
     lazy val b4 = getSumWalletsBalance
     lazy val ut = getUTXOSBalance()
-    lazy val wb = getAllWalletBalances.filter(p=>getWallet(t1._1) contains p._1)
-    lazy val ar = addedReps
-      - (changedReps.values.foldLeft(Set[Hash]())((s,p) => s++p)--changedReps.keys).size
-      + changedReps.keys.size
-    lazy val negativeAddressOption = adsAndBalances.filter(_._2 < 0).map(p => (pri(p._1), nr(p._2))).headOption
-    lazy val negativeWalletOption = repsAndBalances.filter(_._2 < 0).headOption
-    lazy val balanceResults = s"UTXOs ${nr(ut)} ADDs ${nr(b3)} WALLETs ${nr(b4)}"
-    lazy val t1 = negativeWalletOption.getOrElse((Hash.zero(10), 0L))
-
-    // strings to print
-    lazy val negativeWalletString = s"\nWALLET BALANCE: ${str1(wb)} \nCHANGED REPS: ${str3(changedReps)} \nREPS AND BALANCES ${str1(repsAndBalances.filter(p=>getWallet(t1._1) contains p._1))}\n"
     lazy val x1 = getAllWalletBalances.toSeq.sortBy(_._1).toMap
     lazy val x2 = getAllBalances.groupBy(p => getRepresentant(p._1)).map(p => (p._1, p._2.map(_._2).sum)).toSeq.sortBy(_._1).toMap
-    lazy val errors = for {k <- x2.keys; if x2(k) != 0 && x2(k) != x1(k)} yield (k, nr(x2(k)) + " " + nr(x1(k)))
-    lazy val wrongWalletTableString = s"${sx}WRONG ${str4(errors.toMap)}${sx}DIFF: ${b4 - b3}"
 
+    // strings to print
+
+    lazy val errors = for {k <- x2.keys; if x2(k) != 0 && x2(k) != x1(k)} yield (k, nr(x2(k)) + " " + nr(x1(k)))
+    lazy val wrongWalletTableString = s"${sx}WRONG ${str4(errors.toMap)}${sx}DIFF: ${nr(b4 - b3)}${sx}CHANGED${str1(changedAddresses)}"
     lazy val shouldBeClosures = countClosures
     lazy val shouldBeAddresses = countAddresses
 
@@ -131,15 +122,11 @@ object TestExplorer {
       Some(s"___ WRONG UPDATE ___")
     else if (s1 != s2)
       Some(s"___ WRONG CHANGED VALUES ___")
-    else if (negativeAddressOption != None)
-      Some(s"___ NEGATIVE ADDRESS ___")
     else if (ut != b3)
       Some(s"___ WRONG ADDRESS TABLE ___")
     else if (b3 != b4)
       Some(s"___ WRONG WALLET TABLE ___ $wrongWalletTableString")
-    else if (negativeWalletOption != None)
-      Some(s"___ NEGATIVE WALLET ___ $negativeWalletString")
-    else if (!errors.isEmpty) 
+    else if (!errors.isEmpty)
       Some(s"--- WRONG WALLET TABLE --- $wrongWalletTableString")
     else if (shouldBeAddresses != currentStat.total_addresses)
       Some(s"--- WRONG TOTAL ADDRESSES")

From 9839690a55dcb2ba55be9bf6748e44c56cc5b63f Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sun, 3 Jun 2018 08:15:30 +0200
Subject: [PATCH 58/69] Fix update balances

---
 src/main/scala/bge/Explorer.scala             | 35 +++++---
 src/main/scala/bge/core/AddressClosure.scala  |  8 +-
 src/main/scala/db/BitcoinDB.scala             | 79 ++++++++++--------
 src/test/scala/ExplorerIntegrationTests.scala | 49 +++++------
 src/test/scala/TestExplorer.scala             | 81 +++++++++++--------
 5 files changed, 141 insertions(+), 111 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 52e05dd..fbc8268 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -113,20 +113,22 @@ object Explorer extends App with db.BitcoinDB {
   }
 
   def resume = {
-    val bc = blockCount
-    val read = new ResumeBlockReader
 
+    val read = new ResumeBlockReader
     val closure = new ResumeClosure(read.processedBlocks)
-    // move me maybe when we are sure or have better tests
+
+    // FIXME
+    // That check is neccessary due to a bug in updateBalances
+    // Delete checkBalances condition when it is fixed
     val checkBalances =  getSumBalance == getSumWalletsBalance
 
     if (!checkBalances)
       log.info("Error during update of balances or stats .... creating balances and stats")
 
     if (read.changedAddresses.size < balanceUpdateLimit && checkBalances) {
-      val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
+      val (adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
       Some(
-        (repsAndAvailable, adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap, closure.addedAds, closure.addedReps)
+        (adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap, closure.addedAds, closure.addedReps)
       )
     }
     else {
@@ -168,7 +170,7 @@ object Explorer extends App with db.BitcoinDB {
 
     }
 
-    log.info("Look file deleted. BGE shut down correctly!")
+    log.info("Lock file deleted. BGE shut down correctly!")
   }
 
   def getWrongBlock: Option[Int] = {
@@ -197,20 +199,27 @@ object Explorer extends App with db.BitcoinDB {
   }
 
   def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int):
-      (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
+      (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
+
+    val (adsAndBalances,repsAndChanges,  repsAndBalances) = updateBalanceTables(changedAddresses.toMap, changedReps.toMap)
+
+    createRichestLists
 
-    val (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances) = updateBalanceTables(changedAddresses.toMap, changedReps.toMap)
-    insertRichestAddresses
-    insertRichestClosures
     updateStatistics(changedReps,addedAds, addedReps)
 
-    (repsAndAvailable, adsAndBalances,repsAndChanges,  repsAndBalances)
+    (adsAndBalances,repsAndChanges,  repsAndBalances)
   }
 
-  def populateStats = {
-    createBalanceTables
+  def createRichestLists = {
+    var startTime = System.currentTimeMillis
     insertRichestAddresses
     insertRichestClosures
+    log.info("Richest list created in " + (System.currentTimeMillis - startTime) / 1000 + "s")
+  }
+
+  def populateStats = {
+    createBalanceTables
+    createRichestLists
     insertStatistics
   }
 
diff --git a/src/main/scala/bge/core/AddressClosure.scala b/src/main/scala/bge/core/AddressClosure.scala
index 063bccd..189f29b 100644
--- a/src/main/scala/bge/core/AddressClosure.scala
+++ b/src/main/scala/bge/core/AddressClosure.scala
@@ -37,13 +37,11 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
       val hashList = addressesPerTxMap.values map (_ map (p=>Hash(p._2)))
       val nonTrivials = hashList filter (_.length > 1)
       val result = nonTrivials.foldLeft (tree) ((t,l) => insertInputsIntoTree(l,t))
-      log.info("reading " + blocks.length + " blocks from " + blockNo)
+      log.info("Closured " + blocks.length + " blocks from " + blockNo)
       result
     }
 
-    val result = (0 until blockHeights.length by closureReadSize).foldRight(new DisjointSets[Hash](unionFindTable))(addBlocks)
-    log.info("finished generation")
-    result
+    (0 until blockHeights.length by closureReadSize).foldRight(new DisjointSets[Hash](unionFindTable))(addBlocks)
   }
 
   def insertInputsIntoTree(addresses: Iterable[Hash], tree: DisjointSets[Hash]): DisjointSets[Hash] =
@@ -52,11 +50,9 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
     addedTree.union(addresses)
   }
 
-  log.info("applying closure ")
   val timeStart = System.currentTimeMillis
   val startTableSize = unionFindTable.size
   val countSave = saveTree(generateTree)
-
   val totalTime = System.currentTimeMillis - timeStart
 
   log.info("Total of %s addresses added to closures in %s s, %s µs per address" format
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index c32e93e..338bd20 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -165,15 +165,15 @@ trait BitcoinDB {
   def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit = {
     DB withTransaction { implicit session =>
       // delete merged wallets
-      val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ changedReps.keys ++ adsAndBalances.keys).map(Hash.hashToArray)
+      val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ changedReps.keys ++ repsAndBalances.keys ++ adsAndBalances.keys).map(Hash.hashToArray)
       closureBalances.filter(_.address inSet toDelete).delete
       for {
         (balances, table) <- Set((adsAndBalances, balances), (repsAndBalances, closureBalances))
         (address, balance) <- balances
       } {
-        if (balance != 0L)
-          table.insertOrUpdate(Hash.hashToArray(address), balance)
-        else
+        if (balance > 0L)
+          table.insertOrUpdate(Hash.hashToArray(address), balance) 
+        else if (balance == 0)
           table.filter(_.address === Hash.hashToArray(address)).delete
       }
     }
@@ -184,7 +184,7 @@ trait BitcoinDB {
   }
 
   def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]):
-    (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long], collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
+    (collection.immutable.Map[Hash, Long], collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
     val clock = System.currentTimeMillis
     currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
 
@@ -192,60 +192,61 @@ trait BitcoinDB {
       for ((address, change) <- changedAddresses - Hash.zero(0))
          yield (address, getBalance(address) + change)
 
-    // FIXME it produces negative balances and split closures
-    val repsAndChanges: collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
-      .distinct
-      .map(rep => changedReps.find(_._2 contains rep).map(_._1).getOrElse(rep))
-      .distinct
-      .map(r => (r,
-             changedAddresses.filter(x => (changedReps.getOrElse(r, Set())+r) contains x._1).map(_._2).sum))
-      .groupBy(p=>getRepresentant(p._1))
-      .map(p=> (p._1, p._2.map(_._2).sum))
+    def wallet(address: Hash, changedReps: collection.immutable.Map[Hash, Set[Hash]]) = changedReps.filter(_._2 contains address).map(_._1).headOption.getOrElse(address)
+
+    // group balances by rep
+    lazy val repsAndChanges = changedAddresses
+      .groupBy(o => wallet(o._1, changedReps))
+      .map(p => (p._1, p._2.map(q => q._2).sum))
       .toSeq
       .sortBy(_._1)
       .toMap
 
-  // FIXME both maps are wrong
-    val repsAndAvailable: scala.collection.immutable.Map[Hash, Long] = (changedReps ++ (changedAddresses - Hash.zero(0))).map(_._1).toList
-      .distinct
-      .map(p => (p,
-             if (changedReps contains p) (changedReps(p)+p) else collection.immutable.Set(getRepresentant(p))))
-      .groupBy(p=>getRepresentant(p._1))
-      .map(p=> (p._1,
-             getWalletBalances(p._2.foldLeft(Set(Hash.zero(0)))((s,t) => s++t._2) + p._1)))
+    // get current balances from DB with a single query
+    lazy val balances = getClosureBalances(repsAndChanges.keys)
+    // get current representants from DB with a single query
+    val representants = getSomeClosures(repsAndChanges.keys)
+    def representant(a: Hash): Hash = representants.getOrElse(a, a)
+    // get sum from old closure balances
+    def available(s: collection.immutable.Set[Hash]): Long = balances.filter(s contains _._1).map(_._2).sum
+  
+
+    def converted(address: Hash, changedReps: collection.immutable.Map[Hash, Set[Hash]]): collection.immutable.Set[Hash] = changedReps.getOrElse(address, collection.immutable.Set())+address+representant(address)
+
+    val repsAndBalances = repsAndChanges
+      .map(w => (representant(w._1), w._1, w._2))
+      .groupBy(_._1)
+      .map(p=> (p._1, available(p._2.foldLeft(Set(Hash.zero(0)))( (s, n) => s ++ converted(n._1, changedReps)))+p._2.map(_._3).sum))
       .toSeq
       .sortBy(_._1)
       .toMap
 
-  // FIXME keys should always match
-    val repsAndBalances: collection.immutable.Map[Hash, Long] = (repsAndAvailable zip repsAndChanges)
-      .map(p=> /*if (p._1._1 != p._2._1) throw new Error(s"WTF ${getRepresentant(p._1._1)} ${getRepresentant(p._2._1)}") else*/ (p._1._1, p._1._2 + p._2._2))
-
-    // update database
     saveBalances(adsAndBalances, repsAndBalances, changedReps)
 
-    log.info("%s balances updated in %s s, %s µs per address "
+    log.info("Balances updated in %s s, %s addresses, %s µs per address "
       format
-               (adsAndBalances.size, (System.currentTimeMillis - clock) / 1000, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
+        ((System.currentTimeMillis - clock) / 1000, adsAndBalances.size, (System.currentTimeMillis - clock) * 1000 / (adsAndBalances.size + 1)))
+
+    (adsAndBalances,repsAndChanges, repsAndBalances)
 
-       (repsAndAvailable, adsAndBalances,repsAndChanges, repsAndBalances)
   }
 
   def insertRichestClosures = {
 
-    var startTime = System.currentTimeMillis
     DB withSession { implicit session =>
+
       val bh = blockCount - 1
       val topClosures = closureBalances.sortBy(_.balance.desc).take(richlistSize).run.toVector
       val topClosuresWithBh = for ((rep, bal) <- topClosures) yield (bh, rep, bal)
+
       richestClosures.insertAll(topClosuresWithBh: _*)
-      log.info("Richest wallets calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s")
+
     }
   }
 
   def insertRichestAddresses = {
 
-    var startTime = System.currentTimeMillis
+    
     DB withSession { implicit session =>
       Q.updateNA("""
        insert
@@ -261,7 +262,7 @@ trait BitcoinDB {
         balance desc
       limit """ + richlistSize + """
     ;""").execute
-      log.info("Richest addresses calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s")
+
     }
   }
 
@@ -466,6 +467,12 @@ trait BitcoinDB {
 
   }
 
+
+
+  def getClosureBalances(a: Iterable[Hash]): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
+    closureBalances.filter(_.address inSetBind a.map(Hash.hashToArray(_))).map(a => (a.address, a.balance)).run.map(p=> (Hash(p._1), p._2)).toMap
+  }
+
   def getWalletBalances(a: collection.immutable.Set[Hash]): Long = DB withSession { implicit session =>
     closureBalances.filter(_.address inSetBind a.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
   }
@@ -482,6 +489,10 @@ trait BitcoinDB {
     addresses.sortBy(_.hash).map(p=>(p.hash, p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
   }
 
+  def getSomeClosures(a: Iterable[Hash]): collection.immutable.Map[Hash, Hash] = DB withSession { implicit session =>
+    addresses.filter(_.hash inSetBind a.map(Hash.hashToArray)).map(p=>(p.hash, p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
+  }
+
   def countClosures(): Int = DB withSession { implicit session =>
     addresses.groupBy(_.representant).map(p => p._1).length.run
   }
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index 3a06644..d87d237 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -1,4 +1,3 @@
-
 import TestExplorer._
 import org.scalatest._
 
@@ -8,9 +7,9 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     startDocker should be (0)
   }
 
-  it should "add 103 blocks and 1 txs to bitcoin" in {
+  it should "create 103 blocks and 5 txs" in {
     addBlocks(102) should be (0)
-    addTxs(1) should be (0)
+    addTxs(5) should be (0)
     addBlocks(1) should be (0)
   }
 
@@ -26,7 +25,7 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     pay(5000) should be (6)
   }
 
-  "explorer" should s"populate $bitcoinCount blocks" in {
+  "explorer" should "populate 104 blocks" in {
     savePopulate should be (None)
   }
 
@@ -41,40 +40,44 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
     saveResume should be (None)
   }
 
-  it should "resume a block with 2 tx" in {
+  it should "resume a block with 7 txs" in {
     addTxs(2) should be (0)
     addBlocks(1) should be (0)
     saveResume should be (None)
   }
 
-  it should "resume 10 blocks with several txs each" in {
-    (1 to 10).foreach(i => {
-      addTxs(5+i) should be (0)
-      addBlocks(1) should be (0)
-    })
+  it should "rollback 3 blocks and resume it again" in {
+    saveRollback(8) should be (None)
     saveResume should be (None)
   }
 
-  it should "rollback 8 blocks and resume it again" in {
+  List((3, 6), (4, 3)).foreach{ case (blocks: Int, txs: Int) => {
+
+    it should s"resume $blocks blocks with $txs txs each" in {
+
+      (1 to blocks).foreach(i => {
+        addTxs(txs) should be (0)
+        addBlocks(1) should be (0)
+      })
+
+      saveResume should be (None)
+
+    }
+
+  }}
+
+  it should "rollback 4 blocks and resume it again" in {
     saveRollback(8) should be (None)
     saveResume should be (None)
   }
 
-  it should "resume a block with 3 tx" in {
-    addTxs(3) should be (0)
+  it should "resume a block with 8 txs" in {
+    addTxs(2) should be (0)
     addBlocks(1) should be (0)
     saveResume should be (None)
   }
 
-  it should s"have saved $bgeCount blocks" in {
-    bitcoinCount should be (bgeCount)
-  }
-
-  it should s"have saved $totalClosures closures" in {
-    bitcoinCount should be (bgeCount)
-  }
-
-  it should s"have saved $totalAddresses addresses" in {
-    bitcoinCount should be (bgeCount)
+  it should s"finish" in {
+    println(s"Created postgres DB with $bgeCount blocks, $totalAddresses addresses and $totalClosures wallets")
   }
 }
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 88295a5..1f32d83 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -25,9 +25,9 @@ object TestExplorer {
 
   private def sp = " "
 
-  private def pri(a: Hash) = a.toString.drop(2).take(4)
+  private def pri(a: Hash) = a.toString//.drop(2).take(4)
 
-  private def str1(n: collection.immutable.Map[Hash, Long]): String = "(" + n.size + ") total: " + nr(n.map(_._2).sum) + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
+  private def str1(n: collection.immutable.Map[Hash, Long]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
 
   private def str2(n: collection.immutable.Map[Hash, Hash]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) +" => "+(pri(m._2))).mkString(sx)
 
@@ -55,30 +55,31 @@ object TestExplorer {
 
   def bgeCount = blockCount
 
-  def totalAddresses = countAddresses
+  def totalAddresses: Int = countAddresses
 
-  def totalClosures = countClosures
+  def totalClosures: Int = countClosures
 
   def stat = currentStat
 
   def saveRollback(i: Int) = {
+
     for (_ <- 0 until i)
       rollBack(blockCount-1)
-    if (blockCount != currentStat.block_height)
-      populateStats
-  
- 
+
+    populateStats
+
     assertBalancesCreatedCorrectly
   }
 
   def saveResume = {
-    val init = currentStat
+
     resume match {
-      case Some((a,b,c,d,e,f,x,y)) =>
-        assertBalancesUpdatedCorrectly(a,b,c,d,e,f,x,y)
+      case Some((a,b,c,d,e,f,x)) =>
+        assertBalancesUpdatedCorrectly(a,b,c,d,e,f,x)
       case None =>
         assertBalancesCreatedCorrectly
     }
+
   }
 
   def savePopulate = {
@@ -86,11 +87,12 @@ object TestExplorer {
     assertBalancesCreatedCorrectly
   }
 
+  var errors = 0
+
   def assertBalancesCreatedCorrectly =
-    assertBalancesUpdatedCorrectly(s,s,s,s,s,x,0,0)
+    assertBalancesUpdatedCorrectly(s,s,s,s,x,0,0)
 
   def assertBalancesUpdatedCorrectly(
-    repsAndAvailable: collection.immutable.Map[Hash, Long],
     adsAndBalances: collection.immutable.Map[Hash, Long],
     repsAndChanges: collection.immutable.Map[Hash, Long],
     changedAddresses: collection.immutable.Map[Hash, Long],
@@ -99,6 +101,10 @@ object TestExplorer {
     addedAdds: Int,
     addedReps: Int): Option[String] = {
 
+    // skip further tests after an error
+
+    if (errors > 0) return Some("___SKIP___")
+
     // values to check
 
     lazy val ca = changedAddresses - Hash.zero(0)
@@ -107,32 +113,37 @@ object TestExplorer {
     lazy val b3 = getSumBalance
     lazy val b4 = getSumWalletsBalance
     lazy val ut = getUTXOSBalance()
-    lazy val x1 = getAllWalletBalances.toSeq.sortBy(_._1).toMap
-    lazy val x2 = getAllBalances.groupBy(p => getRepresentant(p._1)).map(p => (p._1, p._2.map(_._2).sum)).toSeq.sortBy(_._1).toMap
+    lazy val x1 = getAllWalletBalances.toSeq.sortBy(_._1).filter(_._2>0).toMap
+    lazy val x2 = getAllBalances.groupBy(p => getRepresentant(p._1)).map(p => (p._1, p._2.map(_._2).sum)).toSeq.sortBy(_._1).filter(_._2>0).toMap
 
     // strings to print
 
-    lazy val errors = for {k <- x2.keys; if x2(k) != 0 && x2(k) != x1(k)} yield (k, nr(x2(k)) + " " + nr(x1(k)))
-    lazy val wrongWalletTableString = s"${sx}WRONG ${str4(errors.toMap)}${sx}DIFF: ${nr(b4 - b3)}${sx}CHANGED${str1(changedAddresses)}"
-    lazy val shouldBeClosures = countClosures
-    lazy val shouldBeAddresses = countAddresses
+    lazy val wrongClosures = for {k <- x1.keys; if None == x2.get(k) || x2(k) != x1(k)} yield (k, nr(x1(k)))
+
+    lazy val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ changedReps.keys ++ repsAndBalances.keys ++ adsAndBalances.keys - Hash.zero(0)).map(x=>(x,"OUT")).toMap
+
+    lazy val wrongWalletTableString = s"""${sx}DIFF${sx}${nr(b4-b3)}${sx}WRONG ${str4(wrongClosures.toMap)}${sx}CHANGED ${str1(repsAndChanges-Hash.zero(0))}${sx}ADDED ${str1(repsAndBalances-Hash.zero(0))}${sx}DELETED ${str4(toDelete)}"""
 
     // test updateStatistics && test updateBalances
-    if (repsAndBalances.exists(r => r._1 != getRepresentant(r._1)))
-      Some(s"___ WRONG UPDATE ___")
-    else if (s1 != s2)
-      Some(s"___ WRONG CHANGED VALUES ___")
-    else if (ut != b3)
-      Some(s"___ WRONG ADDRESS TABLE ___")
-    else if (b3 != b4)
-      Some(s"___ WRONG WALLET TABLE ___ $wrongWalletTableString")
-    else if (!errors.isEmpty)
-      Some(s"--- WRONG WALLET TABLE --- $wrongWalletTableString")
-    else if (shouldBeAddresses != currentStat.total_addresses)
-      Some(s"--- WRONG TOTAL ADDRESSES")
-    else if (shouldBeClosures != currentStat.total_closures)
-      Some(s"--- WRONG TOTAL WALLETS")
-    else
-      None
+
+    lazy val errorOption =
+      if (x1.exists(r => r._1 != getRepresentant(r._1)))
+        Some(s"___WRONG_CLOSURE_IN_TABLE___$wrongWalletTableString")
+      else if (s1 != s2)
+        Some(s"___WRONG_CHANGED_VALUES___$wrongWalletTableString")
+      else if (ut != b3)
+        Some(s"___WRONG_ADDRESS_TABLE ___$wrongWalletTableString")
+      else if (b3 != b4 || !wrongClosures.isEmpty)
+        Some(s"___WRONG_WALLET_TABLE___$wrongWalletTableString")
+      else if (countAddresses != currentStat.total_addresses)
+        Some(s"___WRONG_TOTAL_ADDRESSES___$wrongWalletTableString")
+      else if (countClosures != currentStat.total_closures)
+        Some(s"___WRONG_TOTAL_WALLETS___$wrongWalletTableString")
+      else
+        None
+
+    errors += (if (errorOption != None) 1 else 0)
+
+    errorOption
   }
 }

From baf294738facd49fa3d60d08156a0f6f52da0a49 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sun, 3 Jun 2018 08:19:32 +0200
Subject: [PATCH 59/69] Clean up unused files

---
 .../webapp/WEB-INF/templates/layouts/default.jade | 12 ------------
 .../WEB-INF/templates/views/hello-scalate.jade    |  4 ----
 src/main/webapp/WEB-INF/web.xml                   | 15 ---------------
 3 files changed, 31 deletions(-)
 delete mode 100644 src/main/webapp/WEB-INF/templates/layouts/default.jade
 delete mode 100644 src/main/webapp/WEB-INF/templates/views/hello-scalate.jade
 delete mode 100644 src/main/webapp/WEB-INF/web.xml

diff --git a/src/main/webapp/WEB-INF/templates/layouts/default.jade b/src/main/webapp/WEB-INF/templates/layouts/default.jade
deleted file mode 100644
index 96d7bd2..0000000
--- a/src/main/webapp/WEB-INF/templates/layouts/default.jade
+++ /dev/null
@@ -1,12 +0,0 @@
--@ val title: String
--@ val headline: String = title
--@ val body: String
-
-!!!
-html
-  head
-    title= title
-  body
-    #content
-      h1= headline
-      != body
diff --git a/src/main/webapp/WEB-INF/templates/views/hello-scalate.jade b/src/main/webapp/WEB-INF/templates/views/hello-scalate.jade
deleted file mode 100644
index 62ef9d6..0000000
--- a/src/main/webapp/WEB-INF/templates/views/hello-scalate.jade
+++ /dev/null
@@ -1,4 +0,0 @@
-- attributes("title") = "Scalatra: a tiny, Sinatra-like web framework for Scala"
-- attributes("headline") = "Welcome to Scalatra"
-
-p= "Hello, Scalate!"
\ No newline at end of file
diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml
deleted file mode 100644
index d8e6377..0000000
--- a/src/main/webapp/WEB-INF/web.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<web-app xmlns="http://java.sun.com/xml/ns/javaee"
-      xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-      xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd"
-      version="3.0">
-
-  <!--
-    This listener loads a class in the default package called ScalatraBootstrap.
-    That class should implement org.scalatra.LifeCycle.  Your app can be
-    configured in Scala code there.
-  -->
-  <listener>
-    <listener-class>org.scalatra.servlet.ScalatraListener</listener-class>
-  </listener>
-</web-app>

From dc9d1e258bc001166809fd36bb1d658b8114a544 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sun, 3 Jun 2018 10:35:59 +0200
Subject: [PATCH 60/69] Minor changes - comments

---
 src/main/scala/db/BitcoinDB.scala | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 338bd20..de7c26c 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -275,10 +275,9 @@ trait BitcoinDB {
 
     DB withSession { implicit session =>
 
-      val query = """
-       delete from stats where block_height = (select coalesce(max(block_height),0) from blocks);
-       insert
-       into stats select
+      List("""delete from stats where block_height = (select coalesce(max(block_height),0) from blocks);""",
+           """
+       insert into stats select
        (select coalesce(max(block_height),0) from blocks),
        (select coalesce(sum(balance)/100000000,0) from balances),
        (select coalesce(sum(txs),0) from blocks),
@@ -290,10 +289,9 @@ trait BitcoinDB {
        """ + nonDustClosures + """,
        """ + closureGini + """,
        """ + addressGini + """,
-       """ + (System.currentTimeMillis / 1000).toString + """;"""
+       """ + (System.currentTimeMillis / 1000).toString + """;""").foreach( q => (Q.u + q).execute )
 
-      (Q.u + query).execute
-      log.info("Stat calculated in " + (System.currentTimeMillis - startTime) / 1000 + "s");
+      log.info("Stat created in " + (System.currentTimeMillis - startTime) / 1000 + "s");
 
     }
   }
@@ -320,8 +318,9 @@ trait BitcoinDB {
       stat.total_transactions = blockDB.map(_.txs).filter(_ > 0).sum.run.getOrElse(0).toLong
       stat.total_bitcoins_in_addresses = balances.map(_.balance).sum.run.getOrElse(0L) / 100000000
       stat.total_addresses += addedAds
-      stat.total_closures += addedClosures // FIXME after a rollback it seems to be wrong.
+      stat.total_closures += addedClosures
       saveStat(stat)
+
       log.info("Stat updated in " + (System.currentTimeMillis - time) / 1000 + " seconds")
     }
   }

From 1273947dfe0bc8c604db6606ce847faabd11cfc4 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Mon, 4 Jun 2018 12:06:45 +0200
Subject: [PATCH 61/69] update sbt-native-packager plugin

---
 project/plugins.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/project/plugins.sbt b/project/plugins.sbt
index 8201f79..c2374bd 100644
--- a/project/plugins.sbt
+++ b/project/plugins.sbt
@@ -1,3 +1,3 @@
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.3")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.4")

From e99eeaf8bf05601a63f4ee7dd28dd6ce511f23b6 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Mon, 4 Jun 2018 12:07:09 +0200
Subject: [PATCH 62/69] add ensime.sbt file outside git for random ensime/sbt
 fixes

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index d802fd8..60a41df 100755
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,4 @@ project/plugins/
 src/main/resources/application.conf
 .history
 blockchain
+ensime.sbt
\ No newline at end of file

From b57000c13a7ad27b9c6ea98d2dd277da4ab7761a Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Mon, 4 Jun 2018 12:10:14 +0200
Subject: [PATCH 63/69] fix method names

---
 src/test/scala/ExplorerIntegrationTests.scala | 70 +++++++++++++++----
 src/test/scala/TestExplorer.scala             |  6 +-
 2 files changed, 61 insertions(+), 15 deletions(-)

diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index d87d237..caa7019 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -1,6 +1,38 @@
 import TestExplorer._
 import org.scalatest._
 
+////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// The functions "safeRollback", "safePopulate" and "safeResume" are just a wrapper for the original
+// Explorer functions, but adding an exhaustive analysis of all the generated data (in case of
+// updateStatistis - updateBalances).
+//
+// Definitions can be found in TestExplorer.
+//
+// After each operation, several data will be checked:
+//
+// UPDATE
+//
+// consistency of repsAndBalances
+// consistency of repsAndChanges
+// consistency of addedReps
+// consistency of addedAdds
+// consistency of changedAddresses
+// consistency of adsAndBalances
+// consistency of changedReps
+// total balances of wallets against addresses and utxos
+// total_closures and total_addresses are the same as with createStatistics
+//
+// CREATE // ROLLBACK
+//
+// total balances of wallets against addresses and utxos
+//
+// NOTE: atm there is a bug leading the balances to be different than the "closureBalances".
+// In order to fix the code we need first a test to cover this issue. I could not yet
+// reproduce the error.
+//
+//////////////////////////////////////////////////////////////////////////////////////////////
+
 class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
   "docker" should "start bitcoin and postgres" in {
@@ -9,7 +41,11 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
   it should "create 103 blocks and 5 txs" in {
     addBlocks(102) should be (0)
-    addTxs(5) should be (0)
+    pay(8) should be (0)
+    pay(7) should be (0)
+    pay(6) should be (0)
+    pay(5) should be (0)
+    pay(4) should be (0)
     addBlocks(1) should be (0)
   }
 
@@ -26,29 +62,36 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
   }
 
   "explorer" should "populate 104 blocks" in {
-    savePopulate should be (None)
+    safePopulate should be (None)
   }
 
-  it should "resume an empty block" in {
+  it should "resume a block with 5 txs" in {
+    pay(19)
+    pay(17)
+    pay(8)
+    pay(14)
+    pay(100)
     addBlocks(1) should be (0)
-    saveResume should be (None)
+    safeResume should be (None)
   }
 
   it should "resume a block with 1 tx" in {
     addTxs(1) should be (0)
     addBlocks(1) should be (0)
-    saveResume should be (None)
+    safeResume should be (None)
   }
 
   it should "resume a block with 7 txs" in {
     addTxs(2) should be (0)
     addBlocks(1) should be (0)
-    saveResume should be (None)
+    safeResume should be (None)
   }
 
   it should "rollback 3 blocks and resume it again" in {
-    saveRollback(8) should be (None)
-    saveResume should be (None)
+    val init = stat
+    safeRollback(8) should be (None)
+    safeResume should be (None)
+    init should be (stat)
   }
 
   List((3, 6), (4, 3)).foreach{ case (blocks: Int, txs: Int) => {
@@ -60,24 +103,27 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
         addBlocks(1) should be (0)
       })
 
-      saveResume should be (None)
+      safeResume should be (None)
 
     }
 
   }}
 
   it should "rollback 4 blocks and resume it again" in {
-    saveRollback(8) should be (None)
-    saveResume should be (None)
+    val init = stat
+    safeRollback(8) should be (None)
+    safeResume should be (None)
+    stat should be (init)
   }
 
   it should "resume a block with 8 txs" in {
     addTxs(2) should be (0)
     addBlocks(1) should be (0)
-    saveResume should be (None)
+    safeResume should be (None)
   }
 
   it should s"finish" in {
     println(s"Created postgres DB with $bgeCount blocks, $totalAddresses addresses and $totalClosures wallets")
   }
+
 }
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 1f32d83..11089be 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -61,7 +61,7 @@ object TestExplorer {
 
   def stat = currentStat
 
-  def saveRollback(i: Int) = {
+  def safeRollback(i: Int) = {
 
     for (_ <- 0 until i)
       rollBack(blockCount-1)
@@ -71,7 +71,7 @@ object TestExplorer {
     assertBalancesCreatedCorrectly
   }
 
-  def saveResume = {
+  def safeResume = {
 
     resume match {
       case Some((a,b,c,d,e,f,x)) =>
@@ -82,7 +82,7 @@ object TestExplorer {
 
   }
 
-  def savePopulate = {
+  def safePopulate = {
     populate
     assertBalancesCreatedCorrectly
   }

From 052e3b7790eef9ad447a9dd45d02593c78803573 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Mon, 4 Jun 2018 12:10:33 +0200
Subject: [PATCH 64/69] fix broken asSet function

---
 src/main/scala/util/DisjointSets.scala | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/main/scala/util/DisjointSets.scala b/src/main/scala/util/DisjointSets.scala
index 94823a4..0746ca2 100644
--- a/src/main/scala/util/DisjointSets.scala
+++ b/src/main/scala/util/DisjointSets.scala
@@ -13,7 +13,7 @@ class DisjointSets[A](val elements: Map[A,(Int,A)] = Map[A,(Int,A)]()) extends D
   }
 
   def asSet: Set[Set[A]] = {
-    val setOfMaps = elements.groupBy(_._2._2).values.toSet
+    val setOfMaps = elements.groupBy(p => onlyFind(p._2._2)).values.toSet
     val setOfSets = setOfMaps map (_.keys.toSet)
     setOfSets
   }

From 26d4a2155896d23b0e3672eebe12450b9f862654 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Sat, 23 Jun 2018 11:11:01 +0200
Subject: [PATCH 65/69] Improve test suite. SoRestructure updateBalances.
 Please squash before merge.

---
 src/main/scala/bge/Explorer.scala             |  17 +-
 .../scala/bge/actions/PopulateClosure.scala   |  10 +-
 .../scala/bge/actions/ResumeClosure.scala     | 112 ++++-------
 src/main/scala/bge/core/AddressClosure.scala  |  41 ++--
 src/main/scala/bge/core/FastBlockReader.scala |   2 +-
 src/main/scala/db/BitcoinDB.scala             | 175 ++++++++++--------
 src/main/scala/util/package.scala             |   8 -
 src/test/scala/ExplorerIntegrationTests.scala |  31 +++-
 src/test/scala/TestExplorer.scala             | 121 +++++++-----
 9 files changed, 275 insertions(+), 242 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index fbc8268..45cc033 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -1,7 +1,7 @@
 import actions._
 import util._
 import sys.process._
-import collection.mutable.Map
+
 
 /**
  * Created with IntelliJ IDEA.
@@ -115,7 +115,7 @@ object Explorer extends App with db.BitcoinDB {
   def resume = {
 
     val read = new ResumeBlockReader
-    val closure = new ResumeClosure(read.processedBlocks)
+    val closure = new ResumeClosure(read.processedBlocks, read.changedAddresses.toMap)
 
     // FIXME
     // That check is neccessary due to a bug in updateBalances
@@ -126,9 +126,10 @@ object Explorer extends App with db.BitcoinDB {
       log.info("Error during update of balances or stats .... creating balances and stats")
 
     if (read.changedAddresses.size < balanceUpdateLimit && checkBalances) {
-      val (adsAndBalances,repsAndChanges,  repsAndBalances) = resumeStats(read.changedAddresses, convertToMap(closure.changedReps), closure.addedAds, closure.addedReps)
+      val (adsAndBalances,repsAndChanges,  repsAndBalances) =
+        resumeStats(read.changedAddresses.toMap, closure.touchedReps, closure.changedReps, closure.addedAds, closure.addedClosures)
       Some(
-        (adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, convertToMap(closure.changedReps).toMap, closure.addedAds, closure.addedReps)
+        (adsAndBalances,repsAndChanges,  read.changedAddresses.toMap, repsAndBalances, closure.touchedReps, closure.changedReps, closure.addedAds, closure.addedClosures)
       )
     }
     else {
@@ -198,14 +199,14 @@ object Explorer extends App with db.BitcoinDB {
 
   }
 
-  def resumeStats(changedAddresses: Map[Hash,Long], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedReps: Int):
-      (collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
+  def resumeStats(changedAddresses: Map[Hash,Long], touchedReps: Map[Hash,Set[Hash]], changedReps: Map[Hash,Set[Hash]], addedAds: Int, addedClosures: Int):
+      (Map[Hash, Long],Map[Hash, Long],Map[Hash, Long]) = {
 
-    val (adsAndBalances,repsAndChanges,  repsAndBalances) = updateBalanceTables(changedAddresses.toMap, changedReps.toMap)
+    val (adsAndBalances,repsAndChanges,  repsAndBalances) = updateBalanceTables(changedAddresses, touchedReps, changedReps)
 
     createRichestLists
 
-    updateStatistics(changedReps,addedAds, addedReps)
+    updateStatistics(addedAds, addedClosures)
 
     (adsAndBalances,repsAndChanges,  repsAndBalances)
   }
diff --git a/src/main/scala/bge/actions/PopulateClosure.scala b/src/main/scala/bge/actions/PopulateClosure.scala
index 899b895..056a27f 100644
--- a/src/main/scala/bge/actions/PopulateClosure.scala
+++ b/src/main/scala/bge/actions/PopulateClosure.scala
@@ -11,19 +11,16 @@ class PopulateClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHei
   lazy val table = LmdbMap.create("closures")
   override lazy val unionFindTable = new ClosureMap(table)
 
-  def saveTree(tree: DisjointSets[Hash]): Int =
+  def saveTree: Int =
   {
+    val tree = generatedTree
     val timeStart = System.currentTimeMillis
     var queries: Vector[(Array[Byte], Array[Byte])] = Vector()
     val totalElements = tree.elements.size
     var counter = 0
     var counterTotal = 0
-
-//    log.info("Saving tree to database...")
     var counterFinal = 0
-    // tree.elements.keys.foldLeft(tree){(t,value) =>
-    //   val (parentOption, newTree) = tree.find(value)
-    //   for (parent <- parentOption )
+
     table.commit
 
     for (key <- tree.elements.keys)
@@ -50,6 +47,7 @@ class PopulateClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHei
     log.info("Saved to DB until element %s in %s s, %s µs per element" format (counterTotal, (System.currentTimeMillis - timeStart)/1000, (System.currentTimeMillis - timeStart)*1000/(counterTotal+1)))
 
     saveElementsToDatabase(queries, counter)
+    syncAddressesWithUTXOs
 
     table.close
 
diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index 0c85a46..f37423a 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -2,99 +2,61 @@ package actions
 
 import core._
 import db._
-import scala.slick.driver.PostgresDriver.simple._
 import util._
-import util.Hash._
-import collection.mutable.Map
+import Hash._
 
-class ResumeClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHeights: Vector[Int]) with BitcoinDB{
+class ResumeClosure(blockHeights: Vector[Int], changedAds: Map[Hash,Long]) extends AddressClosure(blockHeights: Vector[Int]) with BitcoinDB {
 
   lazy val table = LmdbMap.open("closures")
   override lazy val unionFindTable = new ClosureMap(table)
 
-  override def insertInputsIntoTree(addressList: Iterable[Hash], tree: DisjointSets[Hash]): DisjointSets[Hash] =
-  {
-    val (pairList, tree1) = addressList.foldLeft((List[(Hash,Option[Hash])](),tree)){
-      case ((l,t),a) =>
-        val (repOpt,treex) = t.find(a)
-        (l:+(a,repOpt),treex)
-    }
-
-    val addedTree = addressList.foldLeft(tree1)((t:DisjointSets[Hash],a:Hash) => t.add(a))
-    val tree2 = addedTree.union(addressList)
-
-    // update the postgres DB
-
-    // must be at least 2 addresses
-    assert(addressList.size >= 2, "union of trivial closure")
-    val (representantOpt,result) = tree2 find (addressList.head)
-    val newRep = representantOpt.get
-
-    var addRepFlag = 1
-
-    def recordChangedRep(newRep: Hash, oldRep: Hash) = {
-      // we store in a DisjoinSets the partial changes
-      changedReps = changedReps.add(newRep).add(oldRep).union(Iterable(newRep, oldRep))
+  lazy val changedAddresses = changedAds.keys.toSet - Hash.zero(0)
+  lazy val oldReps = getAddressReps(changedAddresses) // only read from addresses once
+
+  lazy val (finalTree, touchedReps) = // a map of new reps to sets of their addresses that have been touched
+    changedAddresses.foldLeft((generatedTree,Map[Hash,Set[Hash]]())){
+      case ((tree,tR),address) =>
+        val (repOpt, ntree) = tree.find(address)
+        (ntree, repOpt match {
+           case None => tR + (address -> Set(address))
+           case Some (r) => tR + (r -> (tR.getOrElse(r, Set())+address))
+         })
     }
-
-
-    DB withSession { implicit session =>
-
-      for ((address, oldRepOpt) <- pairList.distinct)
-        oldRepOpt match
-        {
-          case None =>
-            insertAddress(address,newRep)
-            recordChangedRep(newRep,address)
-          case Some(oldRep) if (oldRep != newRep) =>
-            // if representant new, update everything that had the old one
-            val updateQuery = for(p <- addresses if p.representant === hashToArray(oldRep)) yield p.representant
-            updateQuery.update(newRep) // TODO: compile query
-            recordChangedRep(newRep, oldRep)
- 
-          case _ => addRepFlag = 0 // one of the elements had this rep before => don't count a new one
-        }
-    }
-
-    addedReps += addRepFlag
-
-    result
-
+  // this includes addresses that are not yet in the tree! otherwise, the below would be easier
+  //  streamOfUnionLists.flatten.flatten.toSet.groupBy(generatedTree.onlyFind)
+  
+  // a map of new reps to the old reps of these addresses, or the addresses themselves as their new rep if they weren't here before
+  lazy val changedReps = for { (rep,ads) <- touchedReps }
+                            yield (rep, ads.map(a => oldReps.getOrElse(a,a)))
+
+  lazy val toSave = for {(rep,ads) <- touchedReps.toSeq
+                         ad <- ads
+                         if ! oldReps.contains(ad)
   }
+  yield (ad,rep)
 
+  lazy val addedAds = toSave.size
 
-  override def saveTree(tree: DisjointSets[Hash]): Int =  // don't replace the postgres DB
-  {
-    val no = tree.elements.size - startTableSize // return the number of new elements in the union-find structure
-    table.close // but don't forget to flush
-    saveAddresses
-    no
+  lazy val addedReps = touchedReps.count {  // the number of reps that were not reps before
+    case (rep,_) => ! oldReps.exists{case (ad,oldR) => rep==oldR}
   }
 
-  lazy val addressBuffer: Map[Hash, Hash] = Map()
+  lazy val deletedReps = changedReps.values.flatten.toSet.filter(p => !changedReps.contains(p) && oldReps.values.toSet.contains(p)).size
 
-  def insertAddress(s: (Hash, Hash)) =
-  {
-    addressBuffer += s
-
-    if (addressBuffer.size >= populateTransactionSize)
-      saveAddresses
-  }
+  lazy val addedClosures = addedReps - deletedReps
 
-  def saveAddresses = {
-    val convertedVector = addressBuffer map (p => (hashToArray(p._1), hashToArray(p._2)))
+  override def saveTree: Int =  // don't replace the postgres DB
+  {
+    val no = finalTree.elements.size - startTableSize // return the number of new elements in the union-find structure
 
-    try{
-      DB withSession (addresses.insertAll(convertedVector.toSeq:_*)(_))
-    }
-    catch {
-      case e: java.sql.BatchUpdateException => throw e.getNextException
-    }
+    updateAddressDB(changedReps)
+    saveAddresses(toSave)
 
-    addedAds += addressBuffer.size
+    table.close // but don't forget to flush
 
-    addressBuffer.clear
+    no
   }
+
 }
 
 
diff --git a/src/main/scala/bge/core/AddressClosure.scala b/src/main/scala/bge/core/AddressClosure.scala
index 189f29b..fbbcd1a 100644
--- a/src/main/scala/bge/core/AddressClosure.scala
+++ b/src/main/scala/bge/core/AddressClosure.scala
@@ -15,28 +15,33 @@ import java.lang.System
 abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
 {
   lazy val unionFindTable: Map[Hash,(Int,Hash)] = Map.empty
-  var addedAds = 0
-  var addedReps = 0
 
-  var temp = new ClosureMap(Map.empty)
-  var changedReps = new DisjointSets[Hash](temp)
-  def saveTree(tree: DisjointSets[Hash]): Int
+  // def unions(blocks: Vector[Int]): Iterable[Seq[Hash]] = {
+  //   val txAndAddressList = txListQuery(blocks)
+  //   val addressesPerTxMap = txAndAddressList.groupBy(p=>Hash(p._1))
+  //   val hashList = addressesPerTxMap.values map (_ map (p=>Hash(p._2)))
+  //   hashList // filter (_.length > 1)
+  // }
 
-  def generateTree: DisjointSets[Hash] =
+  // lazy val streamOfUnionLists: Stream[Iterable[Seq[Hash]]] = (0 until blockHeights.length by closureReadSize).toStream map startIndexToUnions
+
+  // def startIndexToUnions(startIndex: Int): Iterable[Seq[Hash]] = {
+  //   val blocks = blockHeights.slice(startIndex,startIndex+closureReadSize)
+  //   val blockNo = blocks.head
+  //   log.info("Closure working on " + blocks.length + " blocks from " + blockNo)
+  //   unions(blocks)
+  // }
+
+  lazy val generatedTree: DisjointSets[Hash] =
   {
-    // TODO: Same problem as with BlockReader, there we used a function "pre" to initialize the values. 
-    temp = new ClosureMap(Map.empty)
-    changedReps = new DisjointSets[Hash](temp)
-    addedAds = 0;
-    addedReps = 0;
     def addBlocks(startIndex: Int, tree: DisjointSets[Hash]): DisjointSets[Hash] = {
       val blocks = blockHeights.slice(startIndex,startIndex+closureReadSize)
       val blockNo = blocks.head
       val txAndAddressList = txListQuery(blocks)
       val addressesPerTxMap = txAndAddressList.groupBy(p=>Hash(p._1))
       val hashList = addressesPerTxMap.values map (_ map (p=>Hash(p._2)))
-      val nonTrivials = hashList filter (_.length > 1)
-      val result = nonTrivials.foldLeft (tree) ((t,l) => insertInputsIntoTree(l,t))
+      // val nonTrivials = hashList filter (_.length > 1) // premature optimization
+      val result = hashList.foldRight (tree)(insertInputsIntoTree)
       log.info("Closured " + blocks.length + " blocks from " + blockNo)
       result
     }
@@ -44,6 +49,14 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
     (0 until blockHeights.length by closureReadSize).foldRight(new DisjointSets[Hash](unionFindTable))(addBlocks)
   }
 
+  def saveTree: Int
+
+  // lazy val generatedTree: DisjointSets[Hash] =
+  //   streamOfUnionLists.foldRight(new DisjointSets[Hash](unionFindTable)){
+  //     case (unions,tree) =>
+  //       unions.foldRight(tree)(insertInputsIntoTree)
+  //   }
+
   def insertInputsIntoTree(addresses: Iterable[Hash], tree: DisjointSets[Hash]): DisjointSets[Hash] =
   {
     val addedTree = addresses.foldLeft(tree)((t:DisjointSets[Hash],a:Hash) => t add a)
@@ -52,7 +65,7 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
 
   val timeStart = System.currentTimeMillis
   val startTableSize = unionFindTable.size
-  val countSave = saveTree(generateTree)
+  val countSave = saveTree
   val totalTime = System.currentTimeMillis - timeStart
 
   log.info("Total of %s addresses added to closures in %s s, %s µs per address" format
diff --git a/src/main/scala/bge/core/FastBlockReader.scala b/src/main/scala/bge/core/FastBlockReader.scala
index 2120eb0..dafc45b 100644
--- a/src/main/scala/bge/core/FastBlockReader.scala
+++ b/src/main/scala/bge/core/FastBlockReader.scala
@@ -42,7 +42,7 @@ trait FastBlockReader extends BlockReader {
     for (output <- outputsInTransaction(trans)) {
       val addressOption: Option[Hash] = getAddressFromOutput(output: TransactionOutput) match {
          case Some(value) => Some(Hash(value))
-          case None => None
+         case None => None
         }
 
       val value = output.getValue.value
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index de7c26c..390f386 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -1,13 +1,11 @@
 // this has all the database stuff.
-
 package db
 
 import slick.driver.PostgresDriver.simple._
-import slick.jdbc.{StaticQuery => Q}
-import slick.jdbc.meta.MTable
-
+import scala.slick.jdbc.{StaticQuery => Q}
+import scala.slick.jdbc.meta.MTable
 import util._
-import collection.mutable.Map
+import Hash._
 
 trait BitcoinDB {
   def blockDB = TableQuery[Blocks]
@@ -65,7 +63,7 @@ trait BitcoinDB {
   def getLastBlock = DB withSession { implicit session => blockDB.sortBy(_.block_height.desc).map(a => (a.hash, a.block_height)).take(1).run.head }
 
   def txListQuery(blocks: Seq[Int]) = {
-    val emptyArray = Hash.zero(0).array.toArray
+    val emptyArray = hashToArray(zero(0))
     DB withSession { implicit session =>
       val q = for (q <- movements.filter(_.height_out inSet blocks).filter(_.address =!= emptyArray))
         yield (q.spent_in_transaction_hash, q.address)
@@ -106,9 +104,10 @@ trait BitcoinDB {
     }
   }
 
+  
   def createBalanceTables = {
     var clock = System.currentTimeMillis
-    DB withSession { implicit session =>
+    DB withTransaction { implicit session =>
 
       deleteIfExists(balances, closureBalances)
       balances.ddl.create
@@ -123,14 +122,13 @@ trait BitcoinDB {
               select address, sum(balance) from
                 ((select a.representant as address, b.balance as balance from balances b, addresses a where b.address = a.hash and b.address != '\x')
                  UNION ALL
-                (select a.address as address, a.balance as balance from balances a left outer join  addresses b on a.address = b.hash where a.address != '\x' and b.representant is null)) table2
-         group by address;
-      """).execute
+                (select a.address as address, a.balance as balance from balances a left outer join  addresses b on a.address = b.hash where a.address != '\x' and b.representant is null)) table2         group by address;
+      """).execute // TODO: this second  part can be left out when the addresses table contains all addresses
 
        (Q.u + "create index balance_2 on closure_balances(balance)").execute
 
-      Q.updateNA("""delete from balances where balance = 0;""").execute
-      Q.updateNA("""delete from closure_balances where balance = 0;""").execute
+       Q.updateNA("""delete from balances where balance = 0;""").execute // premature optimization
+       Q.updateNA("""delete from closure_balances where balance = 0;""").execute
 
       val b3 = balances.map(_.balance).sum.run.getOrElse(0)
       val b4 = closureBalances.map(_.balance).sum.run.getOrElse(0)
@@ -150,6 +148,11 @@ trait BitcoinDB {
     }
   }
 
+  def syncAddressesWithUTXOs =  DB withSession { implicit session =>
+    val ads = addresses.map(_.hash)
+    addresses.insert(utxo.groupBy(_.address).map(_._1).filterNot(_ in ads).map(p => (p,p)))
+  }
+
   def getSumBalance: Long = DB withSession { implicit session =>
     balances.map(_.balance).sum.run.getOrElse(0)
   }
@@ -162,66 +165,75 @@ trait BitcoinDB {
     utxo.map(_.value).sum.run.getOrElse(0L)
   }
 
-  def saveBalances(adsAndBalances: scala.collection.immutable.Map[Hash, Long], repsAndBalances: scala.collection.immutable.Map[Hash, Long], changedReps: scala.collection.immutable.Map[Hash, Set[Hash]]): Unit = {
+  def saveBalances(adsAndBalances: Map[Hash, Long], repsAndBalances: Map[Hash, Long], oldReps: Iterable[Hash]): Unit = {
     DB withTransaction { implicit session =>
       // delete merged wallets
-      val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ changedReps.keys ++ repsAndBalances.keys ++ adsAndBalances.keys).map(Hash.hashToArray)
+      val toDelete = oldReps.map(hashToArray)
       closureBalances.filter(_.address inSet toDelete).delete
       for {
         (balances, table) <- Set((adsAndBalances, balances), (repsAndBalances, closureBalances))
         (address, balance) <- balances
       } {
-        if (balance > 0L)
-          table.insertOrUpdate(Hash.hashToArray(address), balance) 
+        if (balance > 0L)         //  premature optimization
+          table.insertOrUpdate(hashToArray(address), balance)
         else if (balance == 0)
-          table.filter(_.address === Hash.hashToArray(address)).delete
+          table.filter(_.address === hashToArray(address)).delete
       }
     }
   }
 
   def getRepresentant(a: Hash): Hash = DB withSession { implicit session =>
-    addresses.filter(_.hash === Hash.hashToArray(a)).map(_.representant).run.map(Hash(_)).headOption.getOrElse(a)
+    addresses.filter(_.hash === hashToArray(a)).map(_.representant).run.map(Hash(_)).headOption.getOrElse(a)
   }
 
-  def updateBalanceTables(changedAddresses: collection.immutable.Map[Hash, Long], changedReps: collection.immutable.Map[Hash, Set[Hash]]):
-    (collection.immutable.Map[Hash, Long], collection.immutable.Map[Hash, Long],collection.immutable.Map[Hash, Long]) = {
+  def updateAddressDB(changedReps: Map[Hash, Set[Hash]]) = DB withTransaction { implicit session =>
+    for {(rep,oldReps) <- changedReps
+         if oldReps-rep != Set() // (premature?) optimization
+    }
+    {
+      val updateQuery = for(p <- addresses if p.representant inSet (oldReps-rep).map(hashToArray _)) yield p.representant
+      updateQuery.update(hashToArray(rep)) // TODO: compile query
+    }
+  }
+    
+
+  def saveAddresses(news: Seq[(Hash, Hash)]) = {
+    
+    try{
+      DB withSession {
+        val converted = news.map(p => (hashToArray(p._1),hashToArray(p._2)))
+        addresses.insertAll(converted:_*)(_)}
+    }
+    catch {
+      case e: java.sql.BatchUpdateException => throw e.getNextException
+    }
+    
+  }
+   
+  def updateBalanceTables(changedAddresses: Map[Hash, Long], touchedReps: Map[Hash, Set[Hash]], changedReps: Map[Hash, Set[Hash]]):
+    (Map[Hash, Long], Map[Hash, Long],Map[Hash, Long]) = {
     val clock = System.currentTimeMillis
-    currentStat.total_bitcoins_in_addresses += changedAddresses.map { _._2 }.sum
-
-    val adsAndBalances: scala.collection.immutable.Map[Hash, Long] =
-      for ((address, change) <- changedAddresses - Hash.zero(0))
-         yield (address, getBalance(address) + change)
-
-    def wallet(address: Hash, changedReps: collection.immutable.Map[Hash, Set[Hash]]) = changedReps.filter(_._2 contains address).map(_._1).headOption.getOrElse(address)
-
-    // group balances by rep
-    lazy val repsAndChanges = changedAddresses
-      .groupBy(o => wallet(o._1, changedReps))
-      .map(p => (p._1, p._2.map(q => q._2).sum))
-      .toSeq
-      .sortBy(_._1)
-      .toMap
-
-    // get current balances from DB with a single query
-    lazy val balances = getClosureBalances(repsAndChanges.keys)
-    // get current representants from DB with a single query
-    val representants = getSomeClosures(repsAndChanges.keys)
-    def representant(a: Hash): Hash = representants.getOrElse(a, a)
-    // get sum from old closure balances
-    def available(s: collection.immutable.Set[Hash]): Long = balances.filter(s contains _._1).map(_._2).sum
-  
 
-    def converted(address: Hash, changedReps: collection.immutable.Map[Hash, Set[Hash]]): collection.immutable.Set[Hash] = changedReps.getOrElse(address, collection.immutable.Set())+address+representant(address)
+    val adsAndBalances: Map[Hash, Long] =
+      for ((address, change) <- changedAddresses - zero(0))
+      yield (address, getBalance(address) + change)
+ 
+    lazy val repsAndChanges = for ((rep,ads) <- touchedReps)
+                              yield (rep,ads.toSeq.map(changedAddresses(_)).sum) // the toSeq is necessary, because otherwise equal values get swallowed in the set
+       
+    lazy val oldReps = changedReps.values.flatten
+    lazy val newReps = repsAndChanges.keys
+    // get old closure balances from DB with a single query
+    lazy val oldClosureBalances = getClosureBalances(oldReps++newReps)
+
+    def oldSum(s: Set[Hash]): Long = s.toSeq.map(a => oldClosureBalances.getOrElse(a, getBalance(a))).sum
+
+    def total(rep: Hash) = oldSum(changedReps.getOrElse(rep, Set()))
 
-    val repsAndBalances = repsAndChanges
-      .map(w => (representant(w._1), w._1, w._2))
-      .groupBy(_._1)
-      .map(p=> (p._1, available(p._2.foldLeft(Set(Hash.zero(0)))( (s, n) => s ++ converted(n._1, changedReps)))+p._2.map(_._3).sum))
-      .toSeq
-      .sortBy(_._1)
-      .toMap
+    val repsAndBalances = for ((rep,change) <- repsAndChanges)
+                          yield (rep, total(rep)+change)
 
-    saveBalances(adsAndBalances, repsAndBalances, changedReps)
+    saveBalances(adsAndBalances, repsAndBalances, oldReps)
 
     log.info("Balances updated in %s s, %s addresses, %s µs per address "
       format
@@ -292,19 +304,15 @@ trait BitcoinDB {
        """ + (System.currentTimeMillis / 1000).toString + """;""").foreach( q => (Q.u + q).execute )
 
       log.info("Stat created in " + (System.currentTimeMillis - startTime) / 1000 + "s");
-
     }
   }
 
-  def updateStatistics(changedReps: Map[Hash, Set[Hash]], addedAds: Int, addedReps: Int) = {
+  def updateStatistics(addedAds: Int, addedClosures: Int) = {
 
     val time = System.currentTimeMillis
     val (nonDustAddresses, addressGini) = getGini(balances)
     val (nonDustClosures, closureGini) = getGini(closureBalances)
-    val addedClosures = addedReps
-      - (changedReps.values.foldLeft(Set[Hash]())((s,p) => s++p)--changedReps.keys).size
-      + changedReps.keys.size
-
+    
     DB withSession { implicit session =>
       val stat = currentStat
       stat.total_addresses_with_balance = balances.length.run
@@ -445,51 +453,60 @@ trait BitcoinDB {
     stats.filter(_.block_height === blockHeight).delete
     richestAddresses.filter(_.block_height === blockHeight).delete
     richestClosures.filter(_.block_height === blockHeight).delete
+
     val table = LmdbMap.open("utxos")
     val utxoTable = new UTXOs(table)
-
     val utxoQuery = utxo.filter(_.block_height === blockHeight)
 
     for ((tx, idx) <- utxoQuery.map(p => (p.transaction_hash, p.index)).run)
       utxoTable -= Hash(tx) -> idx
     utxoQuery.delete
-
-    val movementQuery = movements.filter(_.height_out === blockHeight)
-    val utxoRows = movementQuery.filter(_.height_in =!= blockHeight).map(p => (p.transaction_hash, p.address, p.index, p.value, p.height_in)).run
+    // It seems to be a problem here...
+    //val movementQuery = //movements.filter(_.height_out === blockHeight)
+    val movementsToDelete = movements.filter(x => x.height_out === blockHeight || x.height_in === blockHeight)
+    val utxoRows = movements.filter(x => x.height_in === blockHeight || x.height_out===blockHeight).map(p => (p.transaction_hash, p.address, p.index, p.value, p.height_in)).run
+    println(movementsToDelete.map(x => (x.height_out, x.height_in, x.value)).run.map(p=> (p._1,p._2)))
+    println(utxoRows.map(_._5))
     for ((tx, ad, idx, v, h) <- utxoRows)
       utxoTable += ((Hash(tx) -> idx) -> (Hash(ad), v, h))
 
     utxo.insertAll(utxoRows: _*)
-    movementQuery.delete
+    movementsToDelete.delete
     blockDB.filter(_.block_height === blockHeight).delete
     table.close
 
   }
 
+  def getAddressReps(a: Iterable[Hash]): Map[Hash, Hash] = DB withSession {implicit session =>
+    addresses.filter(_.hash inSet (a.map(hashToArray _)) ).map(p => (p.hash,p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
+  }
 
-
-  def getClosureBalances(a: Iterable[Hash]): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
-    closureBalances.filter(_.address inSetBind a.map(Hash.hashToArray(_))).map(a => (a.address, a.balance)).run.map(p=> (Hash(p._1), p._2)).toMap
+  def getClosureBalances(a: Iterable[Hash]): Map[Hash, Long] = DB withSession { implicit session =>
+    closureBalances.filter(_.address inSet a.map(hashToArray(_))).map(a => (a.address, a.balance)).run.map(p=> (Hash(p._1), p._2)).toMap
   }
 
-  def getWalletBalances(a: collection.immutable.Set[Hash]): Long = DB withSession { implicit session =>
-    closureBalances.filter(_.address inSetBind a.map(Hash.hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
+  def getWalletBalances(a: Set[Hash]): Long = DB withSession { implicit session =>
+    closureBalances.filter(_.address inSet a.map(hashToArray(_))).map(_.balance).sum.run.getOrElse(0L)
   }
 
   def getBalance(a: Hash): Long = DB withSession { implicit session =>
-    balances.filter(_.address === Hash.hashToArray(a)).map(_.balance).firstOption.getOrElse(0L)
+    balances.filter(_.address === hashToArray(a)).map(_.balance).firstOption.getOrElse(0L)
   }
 
-  def getAllBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
+  def getAllBalances(): Map[Hash, Long] = DB withSession { implicit session =>
     balances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
   }
 
-  def getClosures(): collection.immutable.Map[Hash, Hash] = DB withSession { implicit session =>
+  def getClosures(): Map[Hash, Hash] = DB withSession { implicit session =>
     addresses.sortBy(_.hash).map(p=>(p.hash, p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
   }
 
-  def getSomeClosures(a: Iterable[Hash]): collection.immutable.Map[Hash, Hash] = DB withSession { implicit session =>
-    addresses.filter(_.hash inSetBind a.map(Hash.hashToArray)).map(p=>(p.hash, p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
+  def getEmptyUTXOs(): Iterable[(Hash)] = DB withSession { implicit session =>
+    utxo.filter(_.value === 0L).map(_.address).run.map(Hash(_))
+  }
+
+  def getAllUTXOs(): Iterable[(Hash, Int)] = DB withSession { implicit session =>
+    utxo.map(p =>(p.address, p.block_height)).run.map(p => (Hash(p._1), p._2))
   }
 
   def countClosures(): Int = DB withSession { implicit session =>
@@ -500,11 +517,15 @@ trait BitcoinDB {
     addresses.length.run
   }
 
-  def getWallet(a: Hash): collection.immutable.Set[Hash] = DB withSession { implicit session =>
-    addresses.filter(_.representant === Hash.hashToArray(a)).map(_.hash).run.map(Hash(_)).toSet
+  def getWallet(a: Hash): Set[Hash] = DB withSession { implicit session =>
+    addresses.filter(_.representant === hashToArray(a)).map(_.hash).run.map(Hash(_)).toSet
   }
 
-  def getAllWalletBalances(): collection.immutable.Map[Hash, Long] = DB withSession { implicit session =>
+  def getAllWalletBalances(): Map[Hash, Long] = DB withSession { implicit session =>
     closureBalances.sortBy(_.address).map(p=>(p.address, p.balance)).run.map(p=>(Hash(p._1),p._2)).toMap
   }
+
+  def deleteLastStats = DB withSession { implicit session =>
+    stats.filter(_.block_height === blockCount - 1).delete
+  }
 }
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 941e6e4..0d7bd1c 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -95,12 +95,4 @@ package object util
       a + (f.getName -> f.get(cc))
     }
 
-  def convertToMap(a: DisjointSets[Hash]): collection.mutable.Map[Hash, Set[Hash]] = {
-    val t: collection.mutable.Map[Hash, Set[Hash]] = collection.mutable.Map.empty
-    for {
-      key <- a.elements.keys
-      parent = a.onlyFind(key)
-    } { t += (parent -> (t.getOrElse(key, Set()) + key))}
-    t
-  }
 }
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index caa7019..a94bc86 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -33,7 +33,7 @@ import org.scalatest._
 //
 //////////////////////////////////////////////////////////////////////////////////////////////
 
-class ExplorerIntegrationTests extends FlatSpec with Matchers {
+class ExplorerIntegrationTests extends FlatSpec with Matchers with CancelGloballyAfterFailure{
 
   "docker" should "start bitcoin and postgres" in {
     startDocker should be (0)
@@ -82,19 +82,33 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
   }
 
   it should "resume a block with 7 txs" in {
-    addTxs(2) should be (0)
+    addTxs(7) should be (0)
     addBlocks(1) should be (0)
     safeResume should be (None)
   }
 
-  it should "rollback 3 blocks and resume it again" in {
+  it should "rollback once and resume it again" in {
     val init = stat
-    safeRollback(8) should be (None)
+    safeRollback() should be (None)
+    safeResume should be (None)
+    init should be (stat)
+  }
+ 
+  it should "resume 2 mores block with 6 and 0 txs" in {
+    addTxs(6) should be (0)
+    addBlocks(2) should be (0)
+    safeResume should be (None)
+  }
+
+  it should "rollback again and resume it again" in {
+    val init = stat
+    safeRollback() should be (None)
     safeResume should be (None)
     init should be (stat)
   }
 
-  List((3, 6), (4, 3)).foreach{ case (blocks: Int, txs: Int) => {
+  List((50, 2), (1, 90
+       )).foreach{ case (blocks: Int, txs: Int) => {
 
     it should s"resume $blocks blocks with $txs txs each" in {
 
@@ -109,15 +123,16 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers {
 
   }}
 
-  it should "rollback 4 blocks and resume it again" in {
+  it should "rollback 8 blocks and resume it again" in {
     val init = stat
-    safeRollback(8) should be (None)
+    println(init)
+    safeRollback() should be (None)
     safeResume should be (None)
     stat should be (init)
   }
 
   it should "resume a block with 8 txs" in {
-    addTxs(2) should be (0)
+    addTxs(8) should be (0)
     addBlocks(1) should be (0)
     safeResume should be (None)
   }
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index 11089be..fcc1f17 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -2,22 +2,23 @@ import util.Hash
 import Explorer._
 import sys.process._
 import scala.util.Random
+import org.scalatest._
 
-object TestExplorer {
+object TestExplorer extends db.BitcoinDB {
 
   // CONSTANTS
 
   val BITCOIN = "docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333 "
 
   val SCRIPTS = "/root/Bitcoin-Graph-Explorer/src/test/docker/"
-
+ 
   val RUNS = 5
 
   // SOME HELPERS
 
-  private def s: collection.immutable.Map[Hash, Long] = collection.immutable.Map()
+  private def s: Map[Hash, Long] = Map()
 
-  private def x:collection.immutable.Map[Hash, Set[Hash]] = collection.immutable.Map()
+  private def x:Map[Hash, Set[Hash]] = Map()
 
   private def nr(b: Long): String = ((b/10000)/10000.0).toString
 
@@ -25,15 +26,15 @@ object TestExplorer {
 
   private def sp = " "
 
-  private def pri(a: Hash) = a.toString//.drop(2).take(4)
+  private def pri(a: Hash) = a.toString.drop(2).take(4)
 
-  private def str1(n: collection.immutable.Map[Hash, Long]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
+  private def str1(n: Map[Hash, Long]): String = "(" + nr(n.map(_._2).sum) + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+(nr(m._2))).mkString(sx)
 
-  private def str2(n: collection.immutable.Map[Hash, Hash]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) +" => "+(pri(m._2))).mkString(sx)
+  private def str2(n: Map[Hash, Hash]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) +" => "+(pri(m._2))).mkString(sx)
 
-  private def str3(n: collection.immutable.Map[Hash, Set[Hash]]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) + " => "+ m._2.map((pri(_))).mkString(" - ")).mkString(sx)
+  private def str3(n: Map[Hash, Set[Hash]]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1) + " => "+ m._2.map((pri(_))).mkString(" - ")).mkString(sx)
 
-  private def str4(n: collection.immutable.Map[Hash, String]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+m._2).mkString(sx)
+  private def str4(n: Map[Hash, String]): String = "(" + n.size + ")" + sx + n.toSeq.sortBy(p=>pri(p._1)).map(m=>pri(m._1)+": "+m._2).mkString(sx)
 
   // BITCOIN-CLI functions
 
@@ -51,6 +52,10 @@ object TestExplorer {
 
   def bitcoinCount: Int = ((BITCOIN + "getblockcount") !!).trim.toInt+1
 
+  def bitcoinTotal: Double = ((BITCOIN + "getbalance") !!).trim.toDouble * 100000000
+
+  def bgeTotal: Long = getSumBalance
+
   // EXPLORER functions
 
   def bgeCount = blockCount
@@ -61,25 +66,22 @@ object TestExplorer {
 
   def stat = currentStat
 
-  def safeRollback(i: Int) = {
-
-    for (_ <- 0 until i)
-      rollBack(blockCount-1)
-
-    populateStats
+  def safeRollback() = {
 
+    //    deleteLastStats
+    // rollback esta roto
+    rollBackToLastStatIfNecessary
     assertBalancesCreatedCorrectly
   }
 
   def safeResume = {
 
     resume match {
-      case Some((a,b,c,d,e,f,x)) =>
-        assertBalancesUpdatedCorrectly(a,b,c,d,e,f,x)
+      case Some((a,b,c,d,e,y,f,x)) =>
+        assertBalancesUpdatedCorrectly(a,b,c,d,e,y,f,x)
       case None =>
         assertBalancesCreatedCorrectly
     }
-
   }
 
   def safePopulate = {
@@ -87,54 +89,63 @@ object TestExplorer {
     assertBalancesCreatedCorrectly
   }
 
-  var errors = 0
-
   def assertBalancesCreatedCorrectly =
-    assertBalancesUpdatedCorrectly(s,s,s,s,x,0,0)
+    assertBalancesUpdatedCorrectly(s,s,s,s,x,x,0,0)
 
   def assertBalancesUpdatedCorrectly(
-    adsAndBalances: collection.immutable.Map[Hash, Long],
-    repsAndChanges: collection.immutable.Map[Hash, Long],
-    changedAddresses: collection.immutable.Map[Hash, Long],
-    repsAndBalances: collection.immutable.Map[Hash, Long],
-    changedReps: collection.immutable.Map[Hash, Set[Hash]],
+    adsAndBalances: Map[Hash, Long],
+    repsAndChanges: Map[Hash, Long],
+    changedAddresses: Map[Hash, Long],
+    repsAndBalances: Map[Hash, Long],
+    touchedReps: Map[Hash, Set[Hash]],
+    changedReps: Map[Hash, Set[Hash]],
     addedAdds: Int,
     addedReps: Int): Option[String] = {
 
-    // skip further tests after an error
-
-    if (errors > 0) return Some("___SKIP___")
-
     // values to check
-
     lazy val ca = changedAddresses - Hash.zero(0)
     lazy val s1 = ca.map(_._2).sum
     lazy val s2 = repsAndChanges.map(_._2).sum
     lazy val b3 = getSumBalance
     lazy val b4 = getSumWalletsBalance
     lazy val ut = getUTXOSBalance()
-    lazy val x1 = getAllWalletBalances.toSeq.sortBy(_._1).filter(_._2>0).toMap
-    lazy val x2 = getAllBalances.groupBy(p => getRepresentant(p._1)).map(p => (p._1, p._2.map(_._2).sum)).toSeq.sortBy(_._1).filter(_._2>0).toMap
+    lazy val x1 = getAllWalletBalances//.filter(_._2>0)
+    lazy val x2 = getAllBalances.groupBy(p => getRepresentant(p._1)).map(p => (p._1, p._2.map(_._2).sum))//.filter(_._2>0)
+    lazy val allClosures = getClosures.groupBy(_._2).map(p => (p._1, p._2.keySet))
+    lazy val emptyUtxos = getEmptyUTXOs()
 
     // strings to print
-
-    lazy val wrongClosures = for {k <- x1.keys; if None == x2.get(k) || x2(k) != x1(k)} yield (k, nr(x1(k)))
-
-    lazy val toDelete = (changedReps.values.fold(Set())((a, b) => a ++ b) ++ changedReps.keys ++ repsAndBalances.keys ++ adsAndBalances.keys - Hash.zero(0)).map(x=>(x,"OUT")).toMap
-
-    lazy val wrongWalletTableString = s"""${sx}DIFF${sx}${nr(b4-b3)}${sx}WRONG ${str4(wrongClosures.toMap)}${sx}CHANGED ${str1(repsAndChanges-Hash.zero(0))}${sx}ADDED ${str1(repsAndBalances-Hash.zero(0))}${sx}DELETED ${str4(toDelete)}"""
+    lazy val wrongClosures = for {k <- x1.keys; if None == x2.get(k) || x2(k) != x1(k)} yield (k, x1(k))
+    lazy val wrongWalletTableString = /*s"""
+        ${sx}TOUCHED REPS${str3(touchedReps)}
+        ${sx}WRONG reps${str1(wrongClosures.toMap)}
+        ${sx}CHANGED ADDs${str1(ca)}
+        ${sx}CHANGED REPRESENTANTS ${str3(changedReps)}
+        ${sx}REPS AND BALANCES${str1(repsAndBalances)}
+        ${sx}COUNT WALLETS CREATE, UPDATE ${countClosures()} ${currentStat.total_closures}
+        ${sx}WALLETS ${str3(allClosures)}
+        ${sx}EMPTY UTXOS ${sx}(${emptyUtxos.size})${sx}${emptyUtxos.map(pri(_))}
+                                       */
+      s"""
+        ${sx}BALANCES ADD - REP - UTXOs${sx}${nr(b3)} ${nr(b4)} ${nr(ut)}
+        ${sx}Stats ${sx}${currentStat}
+        ${sx}UTXOs ${sx}${getAllUTXOs.filter(_._2 == currentStat.block_height).map(p=>pri(p._1)+": "+p._2)}
+"""
+
+    lazy val countsDiffBitcoinToBge = bitcoinTotal - b3
 
     // test updateStatistics && test updateBalances
-
     lazy val errorOption =
-      if (x1.exists(r => r._1 != getRepresentant(r._1)))
+      if (false && emptyUtxos.size > 0)
+        Some(s"___EMPTY UTXOs___$wrongWalletTableString")
+      else if (b3 != b4 || !wrongClosures.isEmpty)
+        Some(s"___WRONG_WALLET_TABLE___$wrongWalletTableString")
+      else if (x1.exists(r => r._1 != getRepresentant(r._1)))
         Some(s"___WRONG_CLOSURE_IN_TABLE___$wrongWalletTableString")
       else if (s1 != s2)
         Some(s"___WRONG_CHANGED_VALUES___$wrongWalletTableString")
       else if (ut != b3)
         Some(s"___WRONG_ADDRESS_TABLE ___$wrongWalletTableString")
-      else if (b3 != b4 || !wrongClosures.isEmpty)
-        Some(s"___WRONG_WALLET_TABLE___$wrongWalletTableString")
       else if (countAddresses != currentStat.total_addresses)
         Some(s"___WRONG_TOTAL_ADDRESSES___$wrongWalletTableString")
       else if (countClosures != currentStat.total_closures)
@@ -142,8 +153,28 @@ object TestExplorer {
       else
         None
 
-    errors += (if (errorOption != None) 1 else 0)
-
     errorOption
   }
 }
+
+object CancelGloballyAfterFailure {
+  @volatile var cancelRemaining = false
+}
+
+trait CancelGloballyAfterFailure extends TestSuiteMixin { this: TestSuite =>
+  import CancelGloballyAfterFailure._
+
+  abstract override def withFixture(test: NoArgTest): Outcome = {
+    if (cancelRemaining)
+      Canceled("Canceled by CancelGloballyAfterFailure because a test failed previously")
+    else
+      super.withFixture(test) match {
+        case failed: Failed =>
+          cancelRemaining = true
+          failed
+        case outcome => outcome
+      }
+  }
+
+  final def newInstance: Suite with OneInstancePerTest = throw new UnsupportedOperationException
+}

From 889f46e8fd677a401e894b356fc35e26af218da3 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Mon, 2 Jul 2018 19:12:46 +0200
Subject: [PATCH 66/69] Reconstruct stats after one or several rollbacks

---
 src/main/scala/bge/Explorer.scala             |  4 ++--
 src/main/scala/db/BitcoinDB.scala             | 14 ++++++++------
 src/test/scala/ExplorerIntegrationTests.scala |  1 -
 src/test/scala/TestExplorer.scala             |  5 +++--
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 45cc033..15a220d 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -155,9 +155,9 @@ object Explorer extends App with db.BitcoinDB {
 
     log.info("Checking for wrong blocks")
 
-    rollBackToLastStatIfNecessary
+    val rollbacked = rollBackToLastStatIfNecessary
 
-    if (newStats)
+    if (rollbacked || newStats )
       populateStats
 
     while (new java.io.File(lockFile).exists) {
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 390f386..280e68f 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -446,6 +446,8 @@ trait BitcoinDB {
 
   def getUtxosMaxHeight = DB withSession { implicit session => utxo.map(_.block_height).max.run.getOrElse(0) }
 
+  // FIXME: rollback do not reconstruct stats, it just delete them and create it again.
+  // It could be added but it does not hurt to create it again (60 minutes at the moment)
   def rollBack(blockHeight: Int) = DB withSession { implicit session =>
 
     log.info("rolling back block " + blockHeight)
@@ -461,12 +463,13 @@ trait BitcoinDB {
     for ((tx, idx) <- utxoQuery.map(p => (p.transaction_hash, p.index)).run)
       utxoTable -= Hash(tx) -> idx
     utxoQuery.delete
-    // It seems to be a problem here...
-    //val movementQuery = //movements.filter(_.height_out === blockHeight)
+    
     val movementsToDelete = movements.filter(x => x.height_out === blockHeight || x.height_in === blockHeight)
-    val utxoRows = movements.filter(x => x.height_in === blockHeight || x.height_out===blockHeight).map(p => (p.transaction_hash, p.address, p.index, p.value, p.height_in)).run
-    println(movementsToDelete.map(x => (x.height_out, x.height_in, x.value)).run.map(p=> (p._1,p._2)))
-    println(utxoRows.map(_._5))
+    val utxoRows = movements
+      .filter(x => x.height_in === blockHeight || x.height_out===blockHeight)
+      .map(p => (p.transaction_hash, p.address, p.index, p.value, p.height_in))
+      .run
+
     for ((tx, ad, idx, v, h) <- utxoRows)
       utxoTable += ((Hash(tx) -> idx) -> (Hash(ad), v, h))
 
@@ -474,7 +477,6 @@ trait BitcoinDB {
     movementsToDelete.delete
     blockDB.filter(_.block_height === blockHeight).delete
     table.close
-
   }
 
   def getAddressReps(a: Iterable[Hash]): Map[Hash, Hash] = DB withSession {implicit session =>
diff --git a/src/test/scala/ExplorerIntegrationTests.scala b/src/test/scala/ExplorerIntegrationTests.scala
index a94bc86..65b267e 100755
--- a/src/test/scala/ExplorerIntegrationTests.scala
+++ b/src/test/scala/ExplorerIntegrationTests.scala
@@ -125,7 +125,6 @@ class ExplorerIntegrationTests extends FlatSpec with Matchers with CancelGloball
 
   it should "rollback 8 blocks and resume it again" in {
     val init = stat
-    println(init)
     safeRollback() should be (None)
     safeResume should be (None)
     stat should be (init)
diff --git a/src/test/scala/TestExplorer.scala b/src/test/scala/TestExplorer.scala
index fcc1f17..e33793a 100644
--- a/src/test/scala/TestExplorer.scala
+++ b/src/test/scala/TestExplorer.scala
@@ -68,9 +68,10 @@ object TestExplorer extends db.BitcoinDB {
 
   def safeRollback() = {
 
-    //    deleteLastStats
-    // rollback esta roto
+    deleteLastStats
+    // rollback do not reconstruct stats, it just delete them and create it again
     rollBackToLastStatIfNecessary
+    populateStats
     assertBalancesCreatedCorrectly
   }
 

From 974b4dc79ce27bd40518feecd2d07127810eb655 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Mon, 2 Jul 2018 19:33:43 +0200
Subject: [PATCH 67/69] Speed up get wrong block

---
 src/main/scala/bge/Explorer.scala | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index 15a220d..d7f7ff0 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -178,6 +178,12 @@ object Explorer extends App with db.BitcoinDB {
 
     val lch = lastCompletedHeight
     val bc = blockCount
+    if (bc-1 > lastCompletedHeight){
+      log.error("Incomplete process, missing stats and maybe more data. Rollback required.")
+      return Some(lch)
+    }
+      
+
     val (count,amount) = sumUTXOs
     val (countDB, amountDB) = countUTXOs
     val expected = totalExpectedSatoshi(bc)

From 5539b708173909b0b5335e2991df21b0509593e9 Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 9 Oct 2018 22:46:09 +0200
Subject: [PATCH 68/69] fix several bugs

---
 build.sbt                                     |  5 +--
 src/main/scala/bge/Explorer.scala             | 32 ++++++++++++-------
 .../scala/bge/actions/PopulateClosure.scala   |  1 -
 .../scala/bge/actions/ResumeClosure.scala     |  3 +-
 src/main/scala/bge/core/AddressClosure.scala  |  8 ++---
 src/main/scala/bge/core/PeerSource.scala      |  4 +--
 src/main/scala/db/BitcoinDB.scala             | 22 +++++++++----
 src/main/scala/util/package.scala             |  4 +--
 8 files changed, 50 insertions(+), 29 deletions(-)

diff --git a/build.sbt b/build.sbt
index 1c4c537..e4ccbea 100644
--- a/build.sbt
+++ b/build.sbt
@@ -62,6 +62,7 @@ case x if Assembly.isConfigFile(x) =>
     MergeStrategy.rename
   case PathList("META-INF", xs @ _*) =>
     (xs map {_.toLowerCase}) match {
+
       case ("manifest.mf" :: Nil) | ("index.list" :: Nil) | ("dependencies" :: Nil) =>
         MergeStrategy.discard
       case ps @ (x :: xs) if ps.last.endsWith(".sf") || ps.last.endsWith(".dsa") =>
@@ -92,8 +93,8 @@ scalacOptions ++= Seq(
 //    ,"-Xlog-implicit-conversions"
 )
 
-javaOptions in run += "-Xmx32G"
-javaOptions in run += "-Xms16"
+javaOptions in run += "-Xmx16G"
+javaOptions in run += "-Xms1G"
 
 fork := true
 
diff --git a/src/main/scala/bge/Explorer.scala b/src/main/scala/bge/Explorer.scala
index d7f7ff0..27739d7 100755
--- a/src/main/scala/bge/Explorer.scala
+++ b/src/main/scala/bge/Explorer.scala
@@ -25,6 +25,9 @@ object Explorer extends App with db.BitcoinDB {
 
       populate
 
+    case "closure"::rest =>
+
+
     case "resume"::rest =>
 
       Seq("touch",lockFile).!
@@ -50,7 +53,13 @@ object Explorer extends App with db.BitcoinDB {
          populate: create the database movements with movements and closures.  Deletes any existing data.
          resume [--newstats]: update the database generated by populate with new incoming data. Works only if populate is already done. The newstats flag forces a regeneration of balance, richlist and stats tables.
          info
-      """)
+      b""")
+  }
+
+  def closure = {
+    new PopulateClosure((1 until blockCount).toVector)
+    createAddressIndexes
+    populateStats
   }
 
   def getInfo = {
@@ -109,6 +118,7 @@ object Explorer extends App with db.BitcoinDB {
     createIndexes
     new PopulateClosure(PopulateBlockReader.processedBlocks)
     createAddressIndexes
+    syncAddressesWithUTXOs
     populateStats
   }
 
@@ -138,15 +148,16 @@ object Explorer extends App with db.BitcoinDB {
     }
   }
 
-  def rollBackToLastStatIfNecessary: Boolean =
-    getWrongBlock match {
-      case None =>
-        false
-      case Some(block: Int) => 
-        rollBack(block)
-        rollBackToLastStatIfNecessary
-        true
+  def rollBackToLastStatIfNecessary: Boolean = {
+    var wrongBlockOption = getWrongBlock
+    var failed = false
+    while (wrongBlockOption != None) {
+      rollBack(wrongBlockOption.get)
+      failed = true
+      wrongBlockOption = getWrongBlock
     }
+    failed
+  }
 
   def iterateResume(newStats: Boolean) = {
 
@@ -180,9 +191,8 @@ object Explorer extends App with db.BitcoinDB {
     val bc = blockCount
     if (bc-1 > lastCompletedHeight){
       log.error("Incomplete process, missing stats and maybe more data. Rollback required.")
-      return Some(lch)
+      return Some(bc-1)
     }
-      
 
     val (count,amount) = sumUTXOs
     val (countDB, amountDB) = countUTXOs
diff --git a/src/main/scala/bge/actions/PopulateClosure.scala b/src/main/scala/bge/actions/PopulateClosure.scala
index 056a27f..7126f23 100644
--- a/src/main/scala/bge/actions/PopulateClosure.scala
+++ b/src/main/scala/bge/actions/PopulateClosure.scala
@@ -47,7 +47,6 @@ class PopulateClosure(blockHeights: Vector[Int]) extends AddressClosure(blockHei
     log.info("Saved to DB until element %s in %s s, %s µs per element" format (counterTotal, (System.currentTimeMillis - timeStart)/1000, (System.currentTimeMillis - timeStart)*1000/(counterTotal+1)))
 
     saveElementsToDatabase(queries, counter)
-    syncAddressesWithUTXOs
 
     table.close
 
diff --git a/src/main/scala/bge/actions/ResumeClosure.scala b/src/main/scala/bge/actions/ResumeClosure.scala
index f37423a..b9b3cb3 100644
--- a/src/main/scala/bge/actions/ResumeClosure.scala
+++ b/src/main/scala/bge/actions/ResumeClosure.scala
@@ -11,7 +11,8 @@ class ResumeClosure(blockHeights: Vector[Int], changedAds: Map[Hash,Long]) exten
   override lazy val unionFindTable = new ClosureMap(table)
 
   lazy val changedAddresses = changedAds.keys.toSet - Hash.zero(0)
-  lazy val oldReps = getAddressReps(changedAddresses) // only read from addresses once
+  //lazy val oldReps = getAddressReps(changedAddresses) // only read from addresses once is dangerous ...
+  lazy val oldReps  = changedAddresses map (add => (add, getRepresentant(add))) toMap
 
   lazy val (finalTree, touchedReps) = // a map of new reps to sets of their addresses that have been touched
     changedAddresses.foldLeft((generatedTree,Map[Hash,Set[Hash]]())){
diff --git a/src/main/scala/bge/core/AddressClosure.scala b/src/main/scala/bge/core/AddressClosure.scala
index fbbcd1a..82e14b1 100644
--- a/src/main/scala/bge/core/AddressClosure.scala
+++ b/src/main/scala/bge/core/AddressClosure.scala
@@ -34,19 +34,19 @@ abstract class AddressClosure(blockHeights: Vector[Int]) extends db.BitcoinDB
 
   lazy val generatedTree: DisjointSets[Hash] =
   {
-    def addBlocks(startIndex: Int, tree: DisjointSets[Hash]): DisjointSets[Hash] = {
+    def addBlocks(tree: DisjointSets[Hash], startIndex: Int): DisjointSets[Hash] = {
       val blocks = blockHeights.slice(startIndex,startIndex+closureReadSize)
       val blockNo = blocks.head
       val txAndAddressList = txListQuery(blocks)
       val addressesPerTxMap = txAndAddressList.groupBy(p=>Hash(p._1))
       val hashList = addressesPerTxMap.values map (_ map (p=>Hash(p._2)))
-      // val nonTrivials = hashList filter (_.length > 1) // premature optimization
-      val result = hashList.foldRight (tree)(insertInputsIntoTree)
+      // val nonTrivials = hashList filter (_.length > 1) // premature optimization?
+      val result = hashList.foldLeft (tree) ((t,l) => insertInputsIntoTree(l,t))
       log.info("Closured " + blocks.length + " blocks from " + blockNo)
       result
     }
 
-    (0 until blockHeights.length by closureReadSize).foldRight(new DisjointSets[Hash](unionFindTable))(addBlocks)
+    (0 until blockHeights.length by closureReadSize).foldLeft(new DisjointSets[Hash](unionFindTable))(addBlocks)
   }
 
   def saveTree: Int
diff --git a/src/main/scala/bge/core/PeerSource.scala b/src/main/scala/bge/core/PeerSource.scala
index 4ab2918..c6c126b 100644
--- a/src/main/scala/bge/core/PeerSource.scala
+++ b/src/main/scala/bge/core/PeerSource.scala
@@ -4,9 +4,9 @@ package core
 import util._
 
 trait PeerSource extends BlockSource {
-  
+
   lazy val truncated = getCurrentLongestChainFromBlockCount take resumeBlockSize
-  
+
   override def blockSource =
   {
     val peer = peerGroup.getConnectedPeers().get(0);
diff --git a/src/main/scala/db/BitcoinDB.scala b/src/main/scala/db/BitcoinDB.scala
index 280e68f..6a64c5a 100644
--- a/src/main/scala/db/BitcoinDB.scala
+++ b/src/main/scala/db/BitcoinDB.scala
@@ -147,10 +147,20 @@ trait BitcoinDB {
       (values.size, values.sum)
     }
   }
-
+  
   def syncAddressesWithUTXOs =  DB withSession { implicit session =>
-    val ads = addresses.map(_.hash)
-    addresses.insert(utxo.groupBy(_.address).map(_._1).filterNot(_ in ads).map(p => (p,p)))
+    // Add addresses that have not yet been spent
+    for {
+      (u,i) <- utxo.map(_.address).run.toVector zipWithIndex } {
+      try {
+        if (i%1000 == 0)
+          log.debug(s"Inserting $i ...")
+        addresses.insert((u,u))
+      }
+      catch {
+        case e: Throwable =>
+      }
+    }
   }
 
   def getSumBalance: Long = DB withSession { implicit session =>
@@ -463,7 +473,7 @@ trait BitcoinDB {
     for ((tx, idx) <- utxoQuery.map(p => (p.transaction_hash, p.index)).run)
       utxoTable -= Hash(tx) -> idx
     utxoQuery.delete
-    
+
     val movementsToDelete = movements.filter(x => x.height_out === blockHeight || x.height_in === blockHeight)
     val utxoRows = movements
       .filter(x => x.height_in === blockHeight || x.height_out===blockHeight)
@@ -480,11 +490,11 @@ trait BitcoinDB {
   }
 
   def getAddressReps(a: Iterable[Hash]): Map[Hash, Hash] = DB withSession {implicit session =>
-    addresses.filter(_.hash inSet (a.map(hashToArray _)) ).map(p => (p.hash,p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
+    addresses.filter(_.hash inSetBind (a.map(hashToArray _)) ).map(p => (p.hash,p.representant)).run.map(p=>(Hash(p._1),Hash(p._2))).toMap
   }
 
   def getClosureBalances(a: Iterable[Hash]): Map[Hash, Long] = DB withSession { implicit session =>
-    closureBalances.filter(_.address inSet a.map(hashToArray(_))).map(a => (a.address, a.balance)).run.map(p=> (Hash(p._1), p._2)).toMap
+    closureBalances.filter(_.address inSetBind a.map(hashToArray(_))).map(a => (a.address, a.balance)).run.map(p=> (Hash(p._1), p._2)).toMap
   }
 
   def getWalletBalances(a: Set[Hash]): Long = DB withSession { implicit session =>
diff --git a/src/main/scala/util/package.scala b/src/main/scala/util/package.scala
index 0d7bd1c..cf1b1ef 100755
--- a/src/main/scala/util/package.scala
+++ b/src/main/scala/util/package.scala
@@ -53,10 +53,10 @@ package object util
 
   def startBitcoinJ: Unit = {
     log.info("starting peergroup")
-
     peerGroup.start
     peerGroup.addAddress(addr)
-    peerGroup.waitForPeers(1).get();
+    peerGroup.waitForPeers(1).get()
+    peerGroup.getDownloadPeer.setDownloadParameters(Long.MaxValue, false) // only download headers
     peerGroup.downloadBlockChain
   }
 

From 6338389c4b1cc51d1877d56fbc7e98f620b52dab Mon Sep 17 00:00:00 2001
From: jorgemartinezpizarro <jorgemartinezpizarro@gmail.com>
Date: Tue, 9 Oct 2018 22:47:49 +0200
Subject: [PATCH 69/69] add missing script

---
 src/test/docker/total.sh | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100755 src/test/docker/total.sh

diff --git a/src/test/docker/total.sh b/src/test/docker/total.sh
new file mode 100755
index 0000000..300bc68
--- /dev/null
+++ b/src/test/docker/total.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+CMD="docker exec --user bitcoin docker_bitcoin_1 bitcoin-cli -regtest -rpcuser=foo -rpcpassword=bar -rpcport=18333 "
+X=$(${CMD} getbalance)
+echo $X