惯性聚合 高效追踪和阅读你感兴趣的博客、新闻、科技资讯
阅读原文 在惯性聚合中打开

推荐订阅源

酷 壳 – CoolShell
酷 壳 – CoolShell
H
Hacker News: Front Page
P
Palo Alto Networks Blog
T
ThreatConnect
Apple Machine Learning Research
Apple Machine Learning Research
博客园_首页
T
True Tiger Recordings
P
Privacy & Cybersecurity Law Blog
B
Blog
IT之家
IT之家
Last Week in AI
Last Week in AI
F
Full Disclosure
Hacker News: Ask HN
Hacker News: Ask HN
C
Comments on: Blog
Microsoft Azure Blog
Microsoft Azure Blog
C
Cybersecurity and Infrastructure Security Agency CISA
Microsoft Security Blog
Microsoft Security Blog
博客园 - 【当耐特】
N
News and Events Feed by Topic
NISL@THU
NISL@THU
腾讯CDC
雷峰网
雷峰网
Security Latest
Security Latest
李成银的技术随笔
M
Microsoft Research Blog - Microsoft Research
L
LangChain Blog
L
Lohrmann on Cybersecurity
cs.CL updates on arXiv.org
cs.CL updates on arXiv.org
C
Check Point Blog
Y
Y Combinator Blog
Recent Announcements
Recent Announcements
博客园 - Franky
N
News | PayPal Newsroom
V
V2EX
A
About on SuperTechFans
The Register - Security
The Register - Security
月光博客
月光博客
奇客Solidot–传递最新科技情报
奇客Solidot–传递最新科技情报
Google Online Security Blog
Google Online Security Blog
MyScale Blog
MyScale Blog
Cisco Talos Blog
Cisco Talos Blog
Vercel News
Vercel News
WordPress大学
WordPress大学
C
Cyber Attacks, Cyber Crime and Cyber Security
The Hacker News
The Hacker News
IntelliJ IDEA : IntelliJ IDEA – the Leading IDE for Professional Development in Java and Kotlin | The JetBrains Blog
IntelliJ IDEA : IntelliJ IDEA – the Leading IDE for Professional Development in Java and Kotlin | The JetBrains Blog
爱范儿
爱范儿
A
Arctic Wolf
L
LINUX DO - 最新话题
freeCodeCamp Programming Tutorials: Python, JavaScript, Git & More

博客园 - 我想我是青蛙

将golang程序注册为windows服务 CentOS上安装spark standalone mode(转载) - 我想我是青蛙 关于听歌这回事 - 我想我是青蛙 c#读取apk 信息 golang 通用Contains方法 - 我想我是青蛙 golang读取文件信息插入mongodb PetaPoco介绍 - 我想我是青蛙 白话MongoDB(三)(转载) - 我想我是青蛙 白话MongoDB(二)(转载) 白话MongoDB(一) (转载) - 我想我是青蛙 给文章加入关键字链接 针对firefox ie6 ie7 ie8的css样式hack (转载) 好久不写日志了,现在开始,好好写了。。 sharepoint 查询calendar recurrence sharepoint之lookup字段 sharepoint获取Audiences 获取exchangeserve的calendar的item sharepoint错误处理 c#上传下载ftp(支持断点续传)
golang 读取mongob数据写入sqlserver
我想我是青蛙 · 2013-06-01 · via 博客园 - 我想我是青蛙

  今天有一个从mongodb读取数据,然后放到sqlserver的工作,当然这等程序必须用go来完成啊。

    先准备mongdb的第三方驱动包 http://labix.org/mgo

      odbc的第三方驱动包 https://bitbucket.org/miquella/mgodbc

  读取mongodb的数据的条件是根据日期范围,以及字符串条件。上代码。

package main

import (
    _ "bitbucket.org/miquella/mgodbc"
    "database/sql"
    "encoding/json"
    "fmt"
    "labix.org/v2/mgo"
    "labix.org/v2/mgo/bson"
    "os"
    "runtime"
    "time"
)

type serverslice struct {
    Servers []string
    Sqlconn string
    Start   string
    End     string
}

type user struct {
    UserName string    "UserName"
    Password string    "Password"
    Email    string    "Email"
    Phone    string    "Phone"
    SubTime  time.Time "SubTime"
}

var config serverslice
var worker = runtime.NumCPU()

//初始化配置
func init() {
    file, _ := os.Open("config.json")
    defer file.Close()
    buf := make([]byte, 2048)

    n, _ := file.Read(buf)
    err := json.Unmarshal(buf[:n], &config)
    if err != nil {
        panic(err)
        fmt.Println(err)
    }
}

func main() {
    runtime.GOMAXPROCS(runtime.NumCPU())

    var chanUser = make(chan user)
    // 标记完成
    dones := make(chan struct{}, worker)

    go readmongodb(chanUser)

    for i := 0; i < worker; i++ {
        go writesql(chanUser, dones)
    }
    awaitForCloseResult(dones)
    fmt.Println("完成")
}

//读取mongodb数据
func readmongodb(chanUser chan<- user) {
    start, _ := time.Parse(layout, config.Start)
    end, _ := time.Parse(layout, config.End)
    for _, server := range config.Servers {
        session, err := mgo.Dial(server)
        if err != nil {
            fmt.Println("打开", server, "失败")
            panic(err)
        }
        defer session.Close()
        var regex = bson.RegEx{}
        regex.Pattern = "^139.*"
        //查询条件是 start<=subtime<=end;email=nil;phone 不以139开头
        var query = bson.M{"SubTime": bson.M{"$gte": start, "$lte": end}, "Email": nil,
            "Phone": bson.M{"$not": regex}}

        //sqlreader类似
        iter := session.DB("db").C("Users").Find(query).Iter()
        message := user{}
        for iter.Next(&message) {
            chanUser <- message
        }
        //关闭通道
        close(chanUser)
    }
}

//写入sqlserver
func writesql(chanUser <-chan user, dones chan<- struct{}) {
    con, err := sql.Open("mgodbc", config.Sqlconn)
    if err != nil {
        fmt.Println(err)
        return
    }
    defer con.Close()
    //con.Exec(prepareSQL)
    for message := range chanUser {
        //组装sql
        var sql = fmt.Sprintf(insertSql, message.UserName, message.Password, message.Email,
            message.Phone, message.SubTime.Format(layout))

        _, err = con.Exec(sql)
        if err != nil {
            fmt.Println(err)
        }
    }
    dones <- struct{}{}
}

/*等待操作完成*/
func awaitForCloseResult(dones <-chan struct{}) {
    for {
        <-dones
        worker--
        if worker <= 0 {
            return
        }
    }
}

var layout = "2006-01-02 15:04:05"

var insertSql string = `INSERT INTO [dbo].[User]
                ([UserName]
                       ,[Password]
                       ,[Email]
                       ,[Phone]
                ,[SubTime] )
                 VALUES
                ('%s','%s','%s','%s','%s');
                `

对应的配置文件config.json

{
    "servers":
    [
        "127.0.0.1:27017"
    ],
    "sqlconn":"driver={SQL Server};SERVER=localhost;UID=aa;PWD=bbb;DATABASE=auth",
    "start":"2013-05-17 00:00:00",
    "end":"2013-06-01 00:00:00"
}