Merge pull request #23 from xssnick/dev-prv

Providers, verification and network improvements

additional/port-check-server/main.go

@@ -13,7 +13,7 @@ func main() {
 	}
 	defer listen.Close()
 
-	println("started on tcp 9099")
+	log.Println("started on tcp 9099")
 	for {
 		conn, err := listen.Accept()
 		if err != nil {
@@ -25,7 +25,7 @@ func main() {
 }
 
 func handle(conn net.Conn) {
-	println("conn from", conn.RemoteAddr().String())
+	log.Println("conn from", conn.RemoteAddr().String())
 	defer conn.Close()
 
 	buffer := make([]byte, 8)
@@ -35,7 +35,7 @@ func handle(conn net.Conn) {
 	}
 
 	if string(buffer[:2]) == "ME" {
-		println("check request from", conn.RemoteAddr().String())
+		log.Println("check request from", conn.RemoteAddr().String())
 
 		conn.Write([]byte("OK"))
 

app.go

@@ -50,6 +50,8 @@ func NewApp() *App {
 	oshook.HookStartup(a.openFile, a.openHash)
 	adnl.Logger = func(v ...any) {}
 	storage.Logger = log.Println
+	storage.DownloadThreads = 120
+	storage.DownloadPrefetch = storage.DownloadThreads * 5
 
 	var err error
 	a.rootPath, err = PrepareRootPath()

build/windows/installer/project.nsi

@@ -22,7 +22,7 @@ Unicode true
 ## !define INFO_PROJECTNAME    "MyProject" # Default "{{.Name}}"
 ## !define INFO_COMPANYNAME    "MyCompany" # Default "{{.Info.CompanyName}}"
 ## !define INFO_PRODUCTNAME    "MyProduct" # Default "{{.Info.ProductName}}"
-## !define INFO_PRODUCTVERSION "1.2.1"     # Default "{{.Info.ProductVersion}}"
+## !define INFO_PRODUCTVERSION "1.3.0"     # Default "{{.Info.ProductVersion}}"
 ## !define INFO_COPYRIGHT      "Copyright" # Default "{{.Info.Copyright}}"
 ###
 ## !define PRODUCT_EXECUTABLE  "Application.exe"      # Default "${INFO_PROJECTNAME}.exe"

core/api/client.go

@@ -10,6 +10,7 @@ import (
 	"github.com/xssnick/tonutils-go/address"
 	"github.com/xssnick/tonutils-go/tlb"
 	"github.com/xssnick/tonutils-storage-provider/pkg/contract"
+	"github.com/xssnick/tonutils-storage/provider"
 	"log"
 	"math/big"
 	"sort"
@@ -99,7 +100,7 @@ type ProviderContract struct {
 	Success   bool
 	Deployed  bool
 	Address   string
-	Providers []Provider
+	Providers []*Provider
 	Balance   string
 }
 
@@ -116,6 +117,7 @@ type Provider struct {
 	Span        string
 	Status      string
 	Reason      string
+	Peer        string
 	Progress    float64
 	Data        NewProviderData
 }
@@ -147,10 +149,10 @@ type StorageClient interface {
 	GetSpeedLimits(ctx context.Context) (*client.SpeedLimits, error)
 	SetSpeedLimits(ctx context.Context, download, upload int64) error
 	GetUploadStats(ctx context.Context, hash []byte) (uint64, error)
-	FetchProviderContract(ctx context.Context, torrentHash []byte, owner *address.Address) (*client.ProviderContractData, error)
-	FetchProviderRates(ctx context.Context, torrentHash, providerKey []byte) (*client.ProviderRates, error)
-	RequestProviderStorageInfo(ctx context.Context, torrentHash, providerKey []byte, owner *address.Address) (*client.ProviderStorageInfo, error)
-	BuildAddProviderTransaction(ctx context.Context, torrentHash []byte, owner *address.Address, providers []client.NewProviderData) (addr *address.Address, bodyData, stateInit []byte, err error)
+	FetchProviderContract(ctx context.Context, torrentHash []byte, owner *address.Address) (*provider.ProviderContractData, error)
+	FetchProviderRates(ctx context.Context, torrentHash, providerKey []byte) (*provider.ProviderRates, error)
+	RequestProviderStorageInfo(ctx context.Context, torrentHash, providerKey []byte, owner *address.Address) (*provider.ProviderStorageInfo, error)
+	BuildAddProviderTransaction(ctx context.Context, torrentHash []byte, owner *address.Address, providers []provider.NewProviderData) (addr *address.Address, bodyData, stateInit []byte, err error)
 	BuildWithdrawalTransaction(torrentHash []byte, owner *address.Address) (addr *address.Address, bodyData []byte, err error)
 	GetNotifier() <-chan bool
 }
@@ -340,6 +342,10 @@ func formatTorrent(full *client.TorrentFull, peersNum int, hide0Speed bool, uplo
 		state = "inactive"
 	}
 
+	if !torrent.Verified {
+		state = "verifying"
+	}
+
 	path := torrent.RootDir
 	if torrent.DirName != nil {
 		path += "/" + *torrent.DirName
@@ -721,7 +727,11 @@ func (a *API) GetProviderContract(hash, ownerAddr string) ProviderContract {
 		return ProviderContract{Success: false}
 	}
 
-	var providers []Provider
+	peers, _ := a.GetPeers(hash)
+
+	var wg sync.WaitGroup
+	var providers []*Provider
+	wg.Add(len(data.Providers))
 	for _, p := range data.Providers {
 		since := "Never"
 		snc := time.Since(p.LastProofAt)
@@ -744,29 +754,51 @@ func (a *API) GetProviderContract(hash, ownerAddr string) ProviderContract {
 			every = fmt.Sprint(p.MaxSpan/86400) + " Days"
 		}
 
-		psi, err := a.client.RequestProviderStorageInfo(a.globalCtx, hashBytes, p.Key, addr)
-		if err != nil {
-			log.Println("failed to request provider info:", err.Error())
-			return ProviderContract{Success: false}
-		}
+		ratePerMB := new(big.Float).SetInt(p.RatePerMB.Nano())
+		szMB := new(big.Float).Quo(new(big.Float).SetUint64(data.Size), big.NewFloat(1024*1024))
+		perDay := new(big.Float).Mul(ratePerMB, szMB)
 
-		providers = append(providers, Provider{
+		perDayNano, _ := perDay.Int(nil)
+		prv := &Provider{
 			Key:         strings.ToUpper(hex.EncodeToString(p.Key)),
 			LastProof:   since,
 			Span:        every,
-			Progress:    psi.Progress,
-			Status:      psi.Status,
-			Reason:      psi.Reason,
-			PricePerDay: tlb.FromNanoTON(new(big.Int).Mul(p.RatePerMB.Nano(), big.NewInt(int64(data.Size/1024/1024)))).String() + " TON",
+			PricePerDay: tlb.FromNanoTON(perDayNano).String() + " TON",
 			Data: NewProviderData{
 				Key:           hex.EncodeToString(p.Key),
 				MaxSpan:       p.MaxSpan,
 				PricePerMBDay: p.RatePerMB.Nano().String(),
 			},
-		})
+		}
+		providers = append(providers, prv)
 
+		go func(p contract.ProviderDataV1) {
+			defer wg.Done()
+
+			psi, err := a.client.RequestProviderStorageInfo(a.globalCtx, hashBytes, p.Key, addr)
+			if err != nil {
+				log.Println("failed to request provider info:", err.Error())
+				prv.Status = "inactive"
+				prv.Reason = "Cannot connect to provider"
+				return
+			}
+
+			prv.Status = psi.Status
+			prv.Progress = psi.Progress
+			prv.Reason = psi.Reason
+
+			for _, peer := range peers {
+				if peer.ADNL == psi.StorageADNL {
+					prv.Peer = psi.StorageADNL[:8]
+					// prv.Reason = "Storage and peer proof received, peer connected: " + prv.Peer[:8] + "..."
+					break
+				}
+			}
+		}(p)
 	}
 
+	wg.Wait()
+
 	bal := data.Balance.String()
 	if idx := strings.IndexByte(bal, '.'); idx != -1 {
 		if len(bal) > idx+4 {
@@ -823,27 +855,32 @@ func (a *API) FetchProviderRates(hash, provider string) ProviderRates {
 		every = fmt.Sprint(span/86400) + " days"
 	}
 
-	ratePerMB := rates.RatePerMBDay.Nano()
-	min := rates.MinBounty.Nano()
-	perDay := new(big.Int).Mul(ratePerMB, big.NewInt(int64(rates.Size/1024/1024)))
+	ratePerMB := new(big.Float).SetInt(rates.RatePerMBDay.Nano())
+	min := new(big.Float).SetInt(rates.MinBounty.Nano())
+
+	szMB := new(big.Float).Quo(new(big.Float).SetUint64(rates.Size), big.NewFloat(1024*1024))
+	perDay := new(big.Float).Mul(ratePerMB, szMB)
 	if perDay.Cmp(min) < 0 {
 		// increase reward to fit min bounty
-		coff := new(big.Float).Quo(new(big.Float).SetInt(min), new(big.Float).SetInt(perDay))
+		coff := new(big.Float).Quo(min, perDay)
 		coff = coff.Add(coff, big.NewFloat(0.01)) // increase a bit to not be less than needed
-		ratePerMB, _ = new(big.Float).Mul(new(big.Float).SetInt(ratePerMB), coff).Int(ratePerMB)
-		perDay = new(big.Int).Mul(ratePerMB, big.NewInt(int64(rates.Size/1024/1024)))
+
+		ratePerMB = new(big.Float).Mul(ratePerMB, coff)
+		perDay = new(big.Float).Mul(ratePerMB, szMB)
 	}
 
+	ratePerMBNano, _ := ratePerMB.Int(nil)
+	perDayNano, _ := perDay.Int(nil)
 	return ProviderRates{
 		Success: true,
 		Provider: Provider{
 			Key:         strings.ToUpper(hex.EncodeToString(providerBytes)),
-			PricePerDay: tlb.FromNanoTON(perDay).String() + " TON",
+			PricePerDay: tlb.FromNanoTON(perDayNano).String() + " TON",
 			Span:        every,
 			Data: NewProviderData{
 				Key:           hex.EncodeToString(providerBytes),
 				MaxSpan:       span,
-				PricePerMBDay: ratePerMB.String(),
+				PricePerMBDay: ratePerMBNano.String(),
 			},
 		},
 	}
@@ -894,7 +931,7 @@ func (a *API) BuildProviderContractData(hash, ownerAddr, amount string, provider
 		return nil, err
 	}
 
-	var prs []client.NewProviderData
+	var prs []provider.NewProviderData
 	for _, p := range providers {
 		keyBytes, err := toHashBytes(p.Key)
 		if err != nil {
@@ -906,7 +943,7 @@ func (a *API) BuildProviderContractData(hash, ownerAddr, amount string, provider
 			return nil, fmt.Errorf("incorrect amount format")
 		}
 
-		prs = append(prs, client.NewProviderData{
+		prs = append(prs, provider.NewProviderData{
 			Address:       address.NewAddress(0, 0, keyBytes),
 			MaxSpan:       p.MaxSpan,
 			PricePerMBDay: tlb.FromNanoTON(price),

core/client/storage.go

@@ -8,6 +8,7 @@ import (
 	"github.com/xssnick/tonutils-go/address"
 	"github.com/xssnick/tonutils-go/liteclient"
 	"github.com/xssnick/tonutils-go/tl"
+	"github.com/xssnick/tonutils-storage/provider"
 	"log"
 	"os"
 	"time"
@@ -329,19 +330,19 @@ func (s *StorageClient) SetSpeedLimits(ctx context.Context, download, upload int
 	return fmt.Errorf("unexpected response")
 }
 
-func (s *StorageClient) FetchProviderContract(ctx context.Context, torrentHash []byte, owner *address.Address) (*ProviderContractData, error) {
+func (s *StorageClient) FetchProviderContract(ctx context.Context, torrentHash []byte, owner *address.Address) (*provider.ProviderContractData, error) {
 	return nil, fmt.Errorf("not supported with storage daemon")
 }
 
-func (s *StorageClient) FetchProviderRates(ctx context.Context, torrentHash, providerKey []byte) (*ProviderRates, error) {
+func (s *StorageClient) FetchProviderRates(ctx context.Context, torrentHash, providerKey []byte) (*provider.ProviderRates, error) {
 	return nil, fmt.Errorf("not supported with storage daemon")
 }
 
-func (s *StorageClient) RequestProviderStorageInfo(ctx context.Context, torrentHash, providerKey []byte, owner *address.Address) (*ProviderStorageInfo, error) {
+func (s *StorageClient) RequestProviderStorageInfo(ctx context.Context, torrentHash, providerKey []byte, owner *address.Address) (*provider.ProviderStorageInfo, error) {
 	return nil, fmt.Errorf("not supported with storage daemon")
 }
 
-func (s *StorageClient) BuildAddProviderTransaction(ctx context.Context, torrentHash []byte, owner *address.Address, providers []NewProviderData) (addr *address.Address, bodyData, stateInit []byte, err error) {
+func (s *StorageClient) BuildAddProviderTransaction(ctx context.Context, torrentHash []byte, owner *address.Address, providers []provider.NewProviderData) (addr *address.Address, bodyData, stateInit []byte, err error) {
 	return nil, nil, nil, fmt.Errorf("not supported with storage daemon")
 }
 

core/client/types.go

@@ -1,17 +1,13 @@
 package client
 
 import (
-	"context"
 	"encoding/binary"
 	"fmt"
-	"github.com/xssnick/tonutils-go/address"
 	"github.com/xssnick/tonutils-go/tl"
 	"github.com/xssnick/tonutils-go/tlb"
 	"github.com/xssnick/tonutils-go/tvm/cell"
-	"github.com/xssnick/tonutils-storage-provider/pkg/contract"
 	"github.com/xssnick/tonutils-storage/storage"
 	"math"
-	"time"
 )
 
 func init() {
@@ -204,6 +200,8 @@ type Torrent struct {
 	DownloadSpeed  float64
 	UploadSpeed    float64
 	FatalError     *string // 2
+
+	Verified bool
 }
 
 type TorrentsList struct {
@@ -217,39 +215,6 @@ type MetaFile struct {
 	Header    *storage.TorrentHeader
 }
 
-type NewProviderData struct {
-	Address       *address.Address
-	MaxSpan       uint32
-	PricePerMBDay tlb.Coins
-}
-
-type ProviderContractData struct {
-	Size      uint64
-	Address   *address.Address
-	Providers []contract.ProviderDataV1
-	Balance   tlb.Coins
-}
-
-type ProviderRates struct {
-	Available        bool
-	RatePerMBDay     tlb.Coins
-	MinBounty        tlb.Coins
-	SpaceAvailableMB uint64
-	MinSpan          uint32
-	MaxSpan          uint32
-
-	Size uint64
-}
-
-type ProviderStorageInfo struct {
-	Status   string
-	Reason   string
-	Progress float64
-
-	Context   context.Context
-	FetchedAt time.Time
-}
-
 func (t *Torrent) Parse(data []byte) (_ []byte, err error) {
 	// Manual parse because of not standard array definition
 	if len(data) < 36 {