package data import ( "context" "fmt" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" "log" "os" "sort" "time" "wss-pool/logging/applogger" "wss-pool/pkg/model" ) var DataBase string var mgoDb *mongo.Client var MgoDbClientMap = map[string]*mongo.Client{} var MgoDbToRedisMap = map[string]string{} type MongoTick struct { Id string `bson:"_id" json:"id"` Channel string `json:"channel"` Amount string `json:"amount"` // 成交量 Count interface{} `bson:"count" json:"count"` // 成交笔数 Open string `json:"open"` // 开盘价 Close string `json:"close"` // 收盘价(当K线为最晚的一根时,是最新成交价) Low string `json:"low"` // 最低价 High string `json:"high"` // 最高价 Vol string `json:"vol"` // 成交额, 即 sum(每一笔成交价 * 该笔的成交量) Timestamp int64 `json:"timestamp"` TradeTurnover string `bson:"trade_turnover" json:"trade_turnover"` Code int64 `json:"code"` } func GetStockKLineTableName(period string) string { return fmt.Sprintf("marketKline%v", period) } func GetContractKLineTableName(period string) string { return fmt.Sprintf("contractKline%v", period) } func GetContractPriceKLineTableName(period string) string { return fmt.Sprintf("contractPriceKline%v", period) } func GetStockUsTableName(period string) string { return fmt.Sprintf("stockUs%v", period) } func GetStockKLineTestTableName(period string) string { return fmt.Sprintf("marketKline%v_test", period) } func GetContractKLineTestTableName(period string) string { return fmt.Sprintf("contractKline%v_test", period) } func GetStockTableName(period string) string { return fmt.Sprintf("stock%v", period) } func GetStockIndexTableName() string { return fmt.Sprintf("stockIndex") } func GetStockIndixKlineTableName(period string) string { return fmt.Sprintf("stockIndex%s", period) } func GetStockSouthAsiaTableName(stock, period string) string { return fmt.Sprintf("stock%s%s", stock, period) } func GetOptionTableName(country string) string { return fmt.Sprintf("option%v", country) } func GetOptionExpiryTableName(country string) string { return fmt.Sprintf("optionExpiry%v", country) } func GetStockList(stockListTable int) { if stockListTable == 0 { StockList = "stockListBak" return } StockList = fmt.Sprintf("stockListBak%d", stockListTable) } // Mgo_init func Mgo_init(config model.Mongodb) { mongodb := config.DbBase GetStockList(config.Table) fmt.Println(StockList) clientOptions := options.Client().ApplyURI(fmt.Sprintf("mongodb://%v:%v@%v:%v/%v?ssl=false&authSource=admin", config.DbUser, config.Password, config.DbHost, config.DbPort, mongodb)) client, err := mongo.Connect(context.TODO(), clientOptions) if err != nil { log.Printf("connect err: %v", err) return } // Check the connection err = client.Ping(context.TODO(), nil) if err != nil { log.Printf("test connect err: %v", err) return } fmt.Println("Mongodb ok") mgoDb = client DataBase = mongodb // init mongo_index Create_stockList_index() Create_stockUs_index() Create_stockIndixList_index() //Create_StockUs_Daily() //Create_StockUs_Weekly() //Create_StockUs_Monthly() //Create_StockUs_Hour() //Create_StockUs_Five_Minutes() //Create_StockUs_One_Minute() Create_SpotKline_index() Create_ContractKline_index() Create_ContractPriceKline_index() Create_StockUsList_index() Create_Stock_index() Create_SpockIndexKline() Create_Stock_News() //测试 //Create_SpotKlineTest_index() //Create_ContractKlineTest_index() //期权 CreateOptionList() CreateOptionindex() // 外汇 CreateForexList() CreateForexListNew() CreateForexTradeLis() CreateForexKLine() } func Mgo_inits(config model.Mongodb) *mongo.Client { mongodb := config.DbBase clientOptions := options.Client().ApplyURI(fmt.Sprintf("mongodb://%v:%v@%v:%v/%v?ssl=false&authSource=admin", config.DbUser, config.Password, config.DbHost, config.DbPort, mongodb)) client, err := mongo.Connect(context.TODO(), clientOptions) if err != nil { log.Printf("connect err: %v", err) return client } // Check the connection err = client.Ping(context.TODO(), nil) if err != nil { log.Printf("test connect err: %v", err) return client } fmt.Println("Mongodb ok") return client } func Mgo_initMap(config model.Mongodb, addrList []string) { GetStockList(config.Table) DataBase = config.DbBase for _, addr := range addrList { clientOptions := options.Client().ApplyURI(fmt.Sprintf("mongodb://%v:%v@%v:%v/%v?ssl=false&authSource=admin", config.DbUser, config.Password, addr, config.DbPort, DataBase)) client, err := mongo.Connect(context.TODO(), clientOptions) if err != nil { log.Printf("connect err: %v", err) return } // Check the connection err = client.Ping(context.TODO(), nil) if err != nil { os.Exit(0) } else { applogger.Info("mongodb init success") } MgoDbClientMap[addr] = client } } // mgoConnect func mgoConnect(collection string) *mongo.Collection { return mgoDb.Database(DataBase).Collection(collection) } // MgoConnect func MgoConnect(collection string) *mongo.Collection { return mgoConnect(collection) } // MgoInsertOne func MgoInsertOne(collection string, docs interface{}) error { c := mgoConnect(collection) if _, err := c.InsertOne(context.TODO(), docs); err != nil { return err } return nil } // MgoInsertMany func MgoInsertMany(collection string, docs []interface{}) error { c := mgoConnect(collection) if _, err := c.InsertMany(context.TODO(), docs); err != nil { return err } return nil } // MgoUpdateID func MgoUpdateID(collection string, id interface{}, update interface{}) error { c := mgoConnect(collection) if _, err := c.UpdateByID(context.TODO(), id, update); err != nil { return err } return nil } // MgoUpdateOne func MgoUpdateOne(collection string, filter interface{}, update interface{}) error { c := mgoConnect(collection) if _, err := c.UpdateOne(context.TODO(), filter, update); err != nil { return err } return nil } func MgoUpdateOneTrue(collection string, filter interface{}, update interface{}) error { c := mgoConnect(collection) opts := options.Update().SetUpsert(true) result, err := c.UpdateOne(context.TODO(), filter, update, opts) if err != nil { return err } if result.MatchedCount == 0 && result.UpsertedCount > 0 { log.Printf("A new document was inserted with the id: %v", result.UpsertedID) } else if result.MatchedCount > 0 { log.Println("An existing document was updated") } else { log.Println("No operation was performed") } return nil } // MgoUpdateMany func MgoUpdateMany(collection string, filter interface{}, update interface{}) error { c := mgoConnect(collection) if _, err := c.UpdateMany(context.TODO(), filter, update); err != nil { return err } return nil } // MgoBulkWrite func MgoBulkWrite(collection string, models []mongo.WriteModel) error { c := mgoConnect(collection) _, err := c.BulkWrite(context.TODO(), models) if err != nil { applogger.Error("UpdateMany err:%v", err) } return err } func MgoBulkWrites(client *mongo.Client, collection string, models []mongo.WriteModel) error { c := client.Database(DataBase).Collection(collection) _, err := c.BulkWrite(context.TODO(), models) if err != nil { applogger.Error("UpdateMany err:%v", err) } return err } // MgoIsExist func MgoIsExist(collection string, filter interface{}) bool { c := mgoConnect(collection) cur, err := c.Find(context.TODO(), filter) if err != nil { return false } var numDocs int for cur.Next(context.Background()) { numDocs++ } return numDocs > 0 } // MgoFind func MgoFind(collection string, filter interface{}) (interface{}, error) { c := mgoConnect(collection) cursor, err := c.Find(context.TODO(), filter, options.Find().SetSort(bson.M{"Vol": -1})) if err != nil { applogger.Error("Find err: %v", err) return nil, err } var results []bson.M if err = cursor.All(context.TODO(), &results); err != nil { log.Fatal(err) } return results, nil } func MgoFindProjectionRes(collection string, filter, projection, sort interface{}, res interface{}, limit int64) { // c := mgoConnect(collection) var optionStr *options.FindOptions if limit > 0 { optionStr = options.Find().SetProjection(projection).SetSort(sort).SetLimit(limit) } else { optionStr = options.Find().SetProjection(projection).SetSort(sort) } cursor, err := c.Find(context.TODO(), filter, optionStr) if err != nil { applogger.Error("Find MgoFindProjectionRes err: %v", err) return } if err = cursor.All(context.TODO(), res); err != nil { log.Fatal(err) } return } func MgoFindProjectionAggregate(collection, field string, filter, projection, sort interface{}, limit int64) []model.StockMogoParam { // //fmt.Println(collection, field, filter) params := make([]model.StockMogoParam, 0) pipeline := mongo.Pipeline{ {{"$match", filter}}, // Convert the string field to a double (floating-point) field {{"$addFields", bson.D{{"stringVal", bson.D{{"$toDouble", "$" + field}}}}}}, // Sort by the new double field {{"$sort", sort}}, // Project only the numeric field // {{"$project", projection}}, // Limit to one document to get the single sorted numeric value {{"$limit", limit}}, } c := mgoConnect(collection) aggregate, err := c.Aggregate(context.TODO(), pipeline) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return params } if aggregate.Next(context.TODO()) { param := model.StockMogoParam{} if err = aggregate.Decode(¶m); err != nil { log.Fatal(err) } params = append(params, param) } //fmt.Printf("%+v", params) return params } func MgoFindRes(collection string, filter interface{}, res interface{}) { c := mgoConnect(collection) cursor, err := c.Find(context.TODO(), filter, options.Find().SetSort(bson.M{"timestamp": 1})) if err != nil { applogger.Error("Find err: %v", err) return } if err = cursor.All(context.TODO(), res); err != nil { log.Fatal(err) } return } func MgoFindStockRes(collection string, filter interface{}, res interface{}) { c := mgoConnect(collection) cursor, err := c.Find(context.TODO(), filter, options.Find().SetSort(bson.M{"Vol": -1})) if err != nil { applogger.Error("Find err: %v", err) return } if err = cursor.All(context.TODO(), res); err != nil { log.Fatal(err) } return } // MgoFindOne func MgoFindOne(collection string, filter interface{}) (bson.M, error) { c := mgoConnect(collection) cur := c.FindOne(context.TODO(), filter) var res bson.M if err := cur.Decode(&res); err != nil { return nil, err } return res, nil } // MgoFindAll func MgoFindAll(collection string, filter interface{}) (interface{}, error) { c := mgoConnect(collection) cur, err := c.Find(context.TODO(), filter) if err != nil { return nil, err } var res interface{} if err := cur.All(context.TODO(), &res); err != nil { return nil, err } return res, err } // MgoDeleteOne func MgoDeleteOne(collection string, filter interface{}) error { c := mgoConnect(collection) _, err := c.DeleteOne(context.TODO(), filter) if err != nil { return err } return nil } // MgoDeleteMany func MgoDeleteMany(collection string, filter interface{}) error { c := mgoConnect(collection) _, err := c.DeleteMany(context.TODO(), filter) if err != nil { return err } return nil } func MillisToTime(millis int64) time.Time { return time.Unix(0, millis*int64(time.Millisecond)) } // MgoDeleteMany100List func MgoDeleteMany100List(collection string, filter interface{}, code string) error { c := mgoConnect(collection) var count int64 = 100 // 查询所有文档并排序 findOptions := options.Find().SetSort(bson.D{{"tick_time", -1}}).SetLimit(count) cursor, err := c.Find(context.Background(), filter, findOptions) if err != nil { applogger.Error("Find err:%v", err) return err } var docs []bson.M if err = cursor.All(context.Background(), &docs); err != nil { return err } if len(docs) < 100 { return nil } //applogger.Debug("总数据:%v", len(docs)) // 获取最新N条记录的创建时间 var latestTimes []time.Time for _, doc := range docs { createdAt := MillisToTime(doc["tick_time"].(int64)) latestTimes = append(latestTimes, createdAt) } // 按时间排序,保证时间最新的在前面 sort.Slice(latestTimes, func(i, j int) bool { return latestTimes[i].After(latestTimes[j]) }) times := latestTimes[count-1].UnixNano() / 1e6 // 转换为毫秒 //applogger.Debug("需要删除的时间节点:%v,时间:%v", latestTimes[count-1], times) // 删除旧的数据,保留最新的N条 deleteFilter := bson.M{ "tick_time": bson.M{ "$lt": times, }, "code": code, } _, err = c.DeleteMany(context.Background(), deleteFilter) if err != nil { return err } //applogger.Debug("Deleted %v documents", deleteResult.DeletedCount) return nil } func MgoFindResM(collection string, filter interface{}) ([]mongo.WriteModel, error) { var results []mongo.WriteModel c := mgoConnect(collection) cur, err := c.Find(context.TODO(), filter, options.Find().SetSort(bson.M{"timestamp": 1})) if err != nil { applogger.Error("Find err: %v", err) return results, err } if err = cur.All(context.TODO(), &results); err != nil { log.Fatal(err) } return results, nil } // MgoPagingFind func MgoPagingFind(collection string, filter interface{}, limit, page int64, sort int) ([]bson.M, error) { c := mgoConnect(collection) var optionStr *options.FindOptions if sort != 0 { optionStr = options.Find().SetLimit(limit).SetSkip((limit * page) - limit).SetSort(bson.M{"Code": sort}) } else { optionStr = options.Find().SetLimit(limit).SetSkip((limit * page) - limit) } fmt.Println(filter) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return nil, err } var results []bson.M if err = cur.All(context.TODO(), &results); err != nil { log.Fatal(err) } return results, err } func MgoPagingFindStruct(collection string, filter interface{}, limit, page int64, sortField string, sort int, res interface{}) error { c := mgoConnect(collection) var optionStr *options.FindOptions optionStr = options.Find().SetLimit(limit).SetSkip((limit * page) - limit).SetSort(bson.M{sortField: sort}) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return err } if err = cur.All(context.TODO(), res); err != nil { log.Fatal(err) return err } return nil } func MgoPagingFindStructList(collection string, filter interface{}, limit, page int64, sortStr string, sort int, res interface{}) error { c := mgoConnect(collection) var optionStr *options.FindOptions optionStr = options.Find().SetLimit(limit).SetSkip((limit * page) - limit).SetSort(bson.M{sortStr: sort}) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return err } if err = cur.All(context.TODO(), res); err != nil { log.Fatal(err) return err } return nil } func MgoPagingFindStructSort(collection string, filter interface{}, limit, page int64, sort interface{}, res interface{}) error { c := mgoConnect(collection) var optionStr *options.FindOptions optionStr = options.Find().SetLimit(limit).SetSkip((limit * page) - limit).SetSort(sort) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return err } if err = cur.All(context.TODO(), res); err != nil { log.Fatal(err) return err } return nil } func MgoPagingFindStructProjection(collection string, filter, projection interface{}, limit, page int64, sort int, res interface{}) error { c := mgoConnect(collection) var optionStr *options.FindOptions optionStr = options.Find().SetLimit(limit).SetSkip((limit * page) - limit).SetSort(bson.M{"Vol": sort}).SetProjection(projection) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return err } if err = cur.All(context.TODO(), res); err != nil { log.Fatal(err) return err } return nil } func MgoPagingFindStructProjectionNew(mongoClient *mongo.Client, collection string, filter, projection interface{}, limit, page int64, sort int, res interface{}) error { // 链接mongodb数据库 c := mongoClient.Database(DataBase).Collection(collection) // 查询mongodb数据 var optionStr *options.FindOptions optionStr = options.Find().SetLimit(limit).SetSkip((limit * page) - limit).SetSort(bson.M{"Vol": sort}).SetProjection(projection) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v,dataTable Name:%v", err, collection) return err } if err = cur.All(context.TODO(), res); err != nil { log.Fatal(err) return err } return nil } func MgoFindToStr(collection string, filter interface{}, limit int64, res interface{}) error { c := mgoConnect(collection) var optionStr *options.FindOptions if limit > 0 { optionStr = options.Find().SetLimit(limit).SetSort(bson.M{"timestamp": -1}) } else { optionStr = options.Find().SetSort(bson.M{"timestamp": -1}) } //fmt.Println(filter) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return err } if err = cur.All(context.TODO(), res); err != nil { log.Fatal(err) return err } return nil } func MgoFindForexToStr(collection string, filter interface{}, limit int64, res interface{}) error { c := mgoConnect(collection) var optionStr *options.FindOptions if limit > 0 { optionStr = options.Find().SetLimit(limit).SetSort(bson.M{"tick_time": -1}) } else { optionStr = options.Find().SetSort(bson.M{"tick_time": -1}) } //fmt.Println(filter) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return err } if err = cur.All(context.TODO(), res); err != nil { log.Fatal(err) return err } return nil } func MgoLimitFind(collection string, filter interface{}, limit int64) ([]MongoTick, error) { res := make([]MongoTick, 0) c := mgoConnect(collection) var optionStr *options.FindOptions if limit > 0 { optionStr = options.Find().SetLimit(limit).SetSort(bson.M{"code": -1}) } else { optionStr = options.Find().SetSort(bson.M{"code": -1}) } //fmt.Println(filter) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return nil, err } //var results []bson.M if err = cur.All(context.TODO(), &res); err != nil { log.Fatal(err) } return res, err } func MgoFinds(collection string, filter interface{}, limit int64) ([]bson.M, error) { c := mgoConnect(collection) var optionStr *options.FindOptions if limit > 0 { optionStr = options.Find().SetLimit(limit).SetSort(bson.M{"timestamp": -1}) } else { optionStr = options.Find().SetSort(bson.M{"timestamp": -1}) } //fmt.Println(filter) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return nil, err } var results []bson.M if err = cur.All(context.TODO(), &results); err != nil { log.Fatal(err) } return results, err } func MgoFindsCode(collection string, filter interface{}, limit int64) ([]bson.M, error) { c := mgoConnect(collection) var optionStr *options.FindOptions if limit > 0 { optionStr = options.Find().SetLimit(limit).SetSort(bson.M{"code": -1}) } else { optionStr = options.Find().SetSort(bson.M{"code": -1}) } //fmt.Println(filter) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return nil, err } var results []bson.M if err = cur.All(context.TODO(), &results); err != nil { log.Fatal(err) } return results, err } func MgoFindProjection(collection string, filter, projection, sort interface{}, limit int64) ([]bson.M, error) { c := mgoConnect(collection) var optionStr *options.FindOptions if limit > 0 { optionStr = options.Find().SetLimit(limit).SetSort(sort).SetProjection(projection) } else { optionStr = options.Find().SetSort(sort).SetProjection(projection) } //fmt.Println(filter) cur, err := c.Find(context.Background(), filter, optionStr) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return nil, err } var results []bson.M if err = cur.All(context.TODO(), &results); err != nil { log.Fatal(err) } return results, err } // MgoFindTotal func MgoFindTotal(collection string, filter interface{}) (int64, error) { c := mgoConnect(collection) total, err := c.CountDocuments(context.TODO(), filter) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return 0, err } return total, nil } // MgoAggregate func MgoAggregate(collection string, filter mongo.Pipeline) (interface{}, error) { c := mgoConnect(collection) aggregate, err := c.Aggregate(context.TODO(), filter) if err != nil { applogger.Error("MgoPagingFind info err:%v", err) return 0, err } var showsWithInfo []map[string]interface{} if err = aggregate.All(context.TODO(), &showsWithInfo); err != nil { log.Printf("collection %s", err) panic(err) } return showsWithInfo, nil }