Fix balances update, make network configurable, add integration tests

.gitignore

@@ -4,8 +4,9 @@ project/target/
 project/plugins/
 .idea/
 *.log
-neodb/
 .ensime
 .ensime_cache/
 src/main/resources/application.conf
-.history
+.history
+blockchain
+ensime.sbt

api/ensime.sbt

@@ -0,0 +1 @@
+ensimeScalaVersion in ThisBuild := "2.11.8"

api/project/build.scala

@@ -11,7 +11,7 @@ import com.earldouglas.xwp.ContainerPlugin.autoImport._
 object BgeapiBuild extends Build {
   val Organization = "net.bitcoinprivacy"
   val Name = "bgeapi"
-  val Version = "3.3"
+  val Version = "3.3-SNAPSHOT"
   val ScalaVersion = "2.11.8"
   val ScalatraVersion = "2.4.0"
 
@@ -24,6 +24,7 @@ object BgeapiBuild extends Build {
       version := Version,
       scalaVersion := ScalaVersion,
       resolvers += Classpaths.typesafeReleases,
+      resolvers += "Local Maven Repository" at "file:///root/.m2/repository",
       resolvers += "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases",
       libraryDependencies ++= Seq(
         "net.bitcoinprivacy" %% "bge" % Version,

api/project/plugins.sbt

@@ -1,6 +1,6 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.3")
 
 addSbtPlugin("com.mojolly.scalate" % "xsbt-scalate-generator" % "0.5.0")
 

api/src/main/scala/api/MyScalatraServlet.scala

@@ -9,12 +9,41 @@ import org.json4s.{DefaultFormats, Formats}
 import org.scalatra.json._
 import net.bitcoinprivacy.bge.models._
 import org.bitcoinj.core.{Address => BitcoinJAddress}
-import org.bitcoinj.params.MainNetParams
+import org.bitcoinj.params._
 import org.scalatra.CorsSupport
+import com.typesafe.config.ConfigFactory
 
 class MyScalatraServlet extends BgeapiStack with db.BitcoinDB with JacksonJsonSupport with CorsSupport {
 
-  // Sets up automatic case class to JSON output serialization, required by
+  lazy val conf = ConfigFactory.load()
+  lazy val network = conf.getString("network")
+
+  lazy val networkParams = network match {
+    case "main" =>
+      MainNetParams.get
+    case "regtest" =>
+      RegTestParams.get
+    case "testnet" =>
+      TestNet3Params.get
+    case _ =>
+      throw new Exception("Unknow params for network"+network)
+
+  }
+
+  case class AddressPrefixes(p2pAddress: String, normalAddress: String)
+  lazy val prefix = network match {
+    case "main" =>
+      AddressPrefixes("05", "00")
+    case "regtest" =>
+      AddressPrefixes("C4", "6F")
+    case "testnet" =>
+      AddressPrefixes("C4", "6F")
+    case _ =>
+      throw new Exception("Unknow params for network"+network)
+
+  }
+
+  // Sets  up automatic case class to JSON output serialization, required by
   // the JValueResult trait.
   protected implicit lazy val jsonFormats: Formats = DefaultFormats
 
@@ -144,13 +173,13 @@ class MyScalatraServlet extends BgeapiStack with db.BitcoinDB with JacksonJsonSu
   def hexAddress(stringAddress: String): String = {
     val arrayAddress = stringAddress.split(",")
     if (arrayAddress.length == 1) {
-      val address = new BitcoinJAddress(MainNetParams.get, stringAddress)
-                                       (if(address.isP2SHAddress) "05" else "00")+valueOf(address.getHash160)
+      val address = new BitcoinJAddress(networkParams, stringAddress)
+      (if(address.isP2SHAddress) prefix.p2pAddress else prefix.normalAddress)+valueOf(address.getHash160)
     }
     else{
       "0" + arrayAddress.length + 
         (for (i <- 0 until arrayAddress.length) 
-         yield  valueOf(new BitcoinJAddress(MainNetParams.get, arrayAddress(i)).getHash160) ).mkString("")
+         yield  valueOf(new BitcoinJAddress(networkParams, arrayAddress(i)).getHash160) ).mkString("")
     }
   }
 

api/src/main/scala/api/models/Address.scala

@@ -2,15 +2,28 @@ package net.bitcoinprivacy.bge.models
 
 import db._
 import org.bitcoinj.core.{Address => Add}
-import org.bitcoinj.params.MainNetParams
+import org.bitcoinj.params._
 import scala.slick.driver.PostgresDriver.simple._
-
+import com.typesafe.config.ConfigFactory
 
 case class Address(address: String, balance: Long)
 case class AddressesSummary(count: Int, sum: Long)
 
 object Address extends db.BitcoinDB {
-
+  lazy val conf = ConfigFactory.load()
+  lazy val network = conf.getString("network")
+
+  lazy val params = network match {
+    case "main" =>
+      MainNetParams.get
+    case "regtest" =>
+      RegTestParams.get
+    case "testnet" =>
+      TestNet3Params.get
+    case _ =>
+      throw new Exception(s"Unknow params for network $network")
+  }
+
   def getWallet(hash: Array[Byte], from: Int, until: Int) = DB withSession { implicit session =>
 
     val repOpt = addresses.filter(_.hash === hash).map(_.representant).firstOption
@@ -80,20 +93,21 @@ object Address extends db.BitcoinDB {
 
   }
 
-  def hashToAddress(hash: Array[Byte]): String = hash.length match {
+  def hashToAddress(hash: Array[Byte]): String = try {hash.length match {
 
-    case 20 => new Add(MainNetParams.get,0,hash).toString
+    case 20 => new Add(params,params.getAddressHeader,hash).toString
 
-    case 21 => new Add(MainNetParams.get,hash.head.toInt,hash.tail).toString
+    case 21 => new Add(params,hash.head.toInt,hash.tail).toString
 
     case 0 => "No decodable address found"
 
     case x if (x%20==1) =>
-
+      try {    
       (for (i <- 1 to hash.length-20 by 20)
        yield hashToAddress(hash.slice(i,i+20)) ).mkString(",")
-
+      } catch { case e:Exception => hash.toString }
     case _  => hash.length + " undefined"
-  }
+    }} catch { case _ => "Bitcoinj failed decoding address" }
+
 
 }

api/src/main/scala/api/models/Stats.scala

@@ -1,19 +1,23 @@
 package net.bitcoinprivacy.bge.models
 
 import scala.slick.driver.PostgresDriver.simple._
+import com.typesafe.config.ConfigFactory
 
-
-case class Stats(block_height:Int, total_bitcoins_in_addresses:Long, total_transactions:Long, total_addresses:Long, total_closures:Long, total_addresses_with_balance:Long, total_closures_with_balance:Long, total_addresses_no_dust:Long, total_closures_no_dust:Long, gini_closure:Double,gini_address:Double, tstamp:Long)
+case class Stats(block_height:Int, total_bitcoins_in_addresses:Long, total_transactions:Long, total_addresses:Long, total_closures:Long, total_addresses_with_balance:Long, total_closures_with_balance:Long, total_addresses_no_dust:Long, total_closures_no_dust:Long, gini_closure:Double,gini_address:Double, tstamp:Long, network:String, dust: Int)
 
 
 object Stats extends db.BitcoinDB
-{ 
+{
+  lazy val conf = ConfigFactory.load()
+  lazy val dustLimit = conf.getInt("dustLimit")
+  lazy val network= conf.getString("network")
+
   def getStats =
     DB withSession { implicit session =>
       val o = stats.sortBy(_.block_height.desc).firstOption
 
       for (p <- o)
-      yield Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12)
+      yield Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12, network, dustLimit)
     }
 
   def getAllStats =
@@ -22,7 +26,7 @@ object Stats extends db.BitcoinDB
       val p1 = stats.sortBy(_.block_height.desc).run.toList
 
       p1 map {
-        p => Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12)
+        p => Stats(p._1,p._2,p._3,p._4,p._5,p._6,p._7,p._8,p._9,p._10,p._11,p._12, network, dustLimit)
       }
     }
 }

build.sbt

@@ -2,25 +2,25 @@ organization := "net.bitcoinprivacy"
 
 name := "bge"
 
-version := "3.3"
+version := "3.3-SNAPSHOT"
 
-scalaVersion := "2.11.8"
+scalaVersion := "2.12.4"
 
 maintainer := "Bitcoinprivacy <info@bitcoinprivacy.net>"
 
 // additional libraries
 libraryDependencies ++= Seq(
-  "org.bitcoinj" % "bitcoinj-core" % "0.13.6",
+  "org.bitcoinj" % "bitcoinj-core" % "0.15-SNAPSHOT",
   "org.xerial.snappy"%"snappy-java"%"1.1.2.4",
   "org.iq80.leveldb"%"leveldb"%"0.7",
   "org.fusesource.leveldbjni"%"leveldbjni-all"%"1.8",
-  "org.postgresql" % "postgresql" % "9.4-1200-jdbc41",
+  "org.postgresql" % "postgresql" % "42.2.2",
   "com.typesafe.slick" %% "slick" % "2.1.0",
   "com.typesafe" % "config" % "1.2.1",
-  "org.scalacheck" %% "scalacheck" % "1.12.4" % "test",
-  "org.scalatest" %% "scalatest" % "2.1.5" % "test",
+  "org.scalacheck" %% "scalacheck" % "1.13.5" % "test",
+  "org.scalatest" %% "scalatest" % "3.2.0-SNAP10" % "test",
   "org.deephacks.lmdbjni" % "lmdbjni" % "0.4.6",
-  "com.typesafe.scala-logging" %% "scala-logging" % "3.4.0",
+  "com.typesafe.scala-logging" %% "scala-logging" % "3.5.0",
   "ch.qos.logback" %  "logback-classic" % "1.1.7",
   ////////////////////////////////////////////////////////////////////
   // select one of the following depending on your architecture
@@ -31,10 +31,15 @@ libraryDependencies ++= Seq(
 )
 
 enablePlugins(JavaAppPackaging)
+enablePlugins(DockerPlugin)
+
+dockerEntrypoint := Seq("bin/bge", "-Dlogback.configurationFile=/conf/logback.xml", "-Dconfig.file=/conf/bge.conf", "--", "start")
+dockerUsername := Some("jorgemartinezpizarro")
+daemonUser in Docker := "root"
 
 libraryDependencies ~= { _.map(_.exclude("org.slf4j", "slf4j-simple")) }
 
-resolvers += "Local Maven Repository" at "file:///"+Path.userHome.absolutePath+"/.m2/repository"
+resolvers += "Local Maven Repository" at "file:///root/.m2/repository"
 
 resolvers += "Fakod Snapshots" at "https://raw.github.com/FaKod/fakod-mvn-repo/master/snapshots"
 
@@ -57,6 +62,7 @@ case x if Assembly.isConfigFile(x) =>
     MergeStrategy.rename
   case PathList("META-INF", xs @ _*) =>
     (xs map {_.toLowerCase}) match {
+
       case ("manifest.mf" :: Nil) | ("index.list" :: Nil) | ("dependencies" :: Nil) =>
         MergeStrategy.discard
       case ps @ (x :: xs) if ps.last.endsWith(".sf") || ps.last.endsWith(".dsa") =>
@@ -91,3 +97,5 @@ javaOptions in run += "-Xmx16G"
 javaOptions in run += "-Xms1G"
 
 fork := true
+
+testOptions in Test += Tests.Argument("-oD")

project/build.properties

@@ -1 +1 @@
-sbt.version=0.13.11
+sbt.version=1.1.0

project/plugins.sbt

@@ -1,3 +1,3 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3")
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 
-addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.2.0")
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.4")

src/main/resources/reference.conf

@@ -11,11 +11,16 @@ levelDBFile = ${dataDir}"level.db"
 lockFile = ${dataDir}"lock"
 dustLimit=546
 databaseName="postgres"
-username = "root"
-password = "trivial"
-host = "localhost"
+username = "user"
+password = "pass"
+host = "localhost:1234"
 jdbcOptions = "?useServerPrepStmts=false&rewriteBatchedStatements=true&maxWait=-1"
 mdb.path = ${dataDir}"mdb/"
 mdb.transactionSize = 100000
-
-logger.scala.slick=ERROR
+network=main # accepts main, testnet, regtest
+logger.scala.slick=ERROR
+resumeBlockSize=1000000 # depending on the available ram
+bitcoin.ip=localhost
+bitcoin.maxPopulate=0 # 0 all,m number block until stop first populate. To rest partial blockchains without running all!
+balanceUpdateLimit=50000 # for balance changes beyond this size, create new balance tables
+checkUTXOsSize=100000