tjzx-spider/main.go

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"strings"

	// 使用纯 Go 的 SQLite 驱动，免除 CGO 依赖
	"github.com/glebarez/sqlite"
	"gorm.io/gorm"
)

// ---- 1. 定义读取 JSON 用的结构体 ----

type JSONResponse struct {
	ResponseCode string   `json:"responseCode"`
	ResponseData RespData `json:"responseData"`
}

type RespData struct {
	Datas []QuestionData `json:"datas"`
}

type QuestionData struct {
	QueId           int              `json:"queId"`
	Stem            string           `json:"stem"`
	QuestionAnswers []QuestionAnswer `json:"questionAnswers"`
}

type QuestionAnswer struct {
	AnswerDesc string `json:"answerDesc"`
	IsCorrect  int    `json:"isCorrect"`
}

// ---- 2. 定义对应 SQLite 的多选题 GORM 模型 ----

type Duoxuan struct {
	ID  int     `gorm:"column:id;primaryKey"`
	Nr  string  `gorm:"column:nr"`
	Da1 *string `gorm:"column:da_1"` // 使用指针类型，当值为 nil 时，写入数据库就是 NULL
	Da2 *string `gorm:"column:da_2"`
	Da3 *string `gorm:"column:da_3"`
	Da4 *string `gorm:"column:da_4"`
}

// TableName 指定绑定的表名为 duoxuan
func (Duoxuan) TableName() string {
	return "duoxuan"
}

func main() {
	// ==================== 第一步：发送多选题的网络请求 ====================
	client := &http.Client{}
	var data = strings.NewReader(`rows=200&page=1&ownerId=229&btId=102404796&queTypeCode=MC&clickQueId=240677024`)
	req, err := http.NewRequest("POST", "https://sia.sinopec.com/ac/seiPracController/querySeiPracQuestionForPage.do", data)
	if err != nil {
		log.Fatalf("创建请求失败: %v", err)
	}

	// 设置 Headers 和 Cookie
	req.Header.Set("Accept", "application/json, text/plain, */*")
	req.Header.Set("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6")
	req.Header.Set("Access-Control-Allow-Origin", "*")
	req.Header.Set("Authorization", "00000-181M")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded;charset=UTF-8")
	req.Header.Set("Origin", "https://sia.sinopec.com")
	req.Header.Set("Referer", "https://sia.sinopec.com/mobile/")
	req.Header.Set("Sec-Fetch-Dest", "empty")
	req.Header.Set("Sec-Fetch-Mode", "cors")
	req.Header.Set("Sec-Fetch-Site", "same-origin")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/150.0.0.0 Safari/537.36 Edg/150.0.0.0")
	req.Header.Set("sec-ch-ua", `"Not;A=Brand";v="8", "Chromium";v="150", "Microsoft Edge";v="150"`)
	req.Header.Set("sec-ch-ua-mobile", "?0")
	req.Header.Set("sec-ch-ua-platform", `"Windows"`)
	req.Header.Set("Cookie", "accessId=839d6250-5041-11ee-96f3-ef2d7900d919; x-forwarded-for=null; zshIo=https%3A%2F%2Fsia.sinopec.com%2Fmobile%2F%23%2Fapp%2Fautonomous%2Ftest%2Fpractice%2Ftopics%3FqbId%3D263%26titleName%3D7%25E7%259F%25B3%25E5%25B7%25A5%25E5%25BB%25BA%252F%25E4%25B8%2593%25E4%25B8%259A%25E7%259F%25A5%25E8%25AF%2586%26domainId%3D6026f6138212ebda9f65e27591ac399f; types=0; Admin-Token=00000-181M; LEARNSSIONID=ccdc9e86-f458-472e-a31c-5ff82876f93f; pageViewNum=4; SERVERID=08454d75353b657bc1b9c1181db254db|1783945192|1783941591")

	resp, err := client.Do(req)
	if err != nil {
		log.Fatalf("发送请求失败: %v", err)
	}
	defer resp.Body.Close()

	bodyText, err := io.ReadAll(resp.Body)
	if err != nil {
		log.Fatalf("读取响应失败: %v", err)
	}

	// ==================== 第二步：解析返回的 JSON ====================
	var apiResponse JSONResponse
	err = json.Unmarshal(bodyText, &apiResponse)
	if err != nil {
		log.Fatalf("JSON 解析失败: %v", err)
	}

	// ==================== 第三步：连接数据库并写入 ====================
	db, err := gorm.Open(sqlite.Open("osecdtzx_question.db"), &gorm.Config{})
	if err != nil {
		log.Fatalf("数据库连接失败: %v", err)
	}

	// 自动检查并迁移表结构
	db.AutoMigrate(&Duoxuan{})

	// 准备批量插入的数据切片
	var rowsToInsert []Duoxuan

	for _, item := range apiResponse.ResponseData.Datas {
		// 1. 收集当前题目所有正确的答案
		var correctAnswers []string
		for _, ans := range item.QuestionAnswers {
			if ans.IsCorrect == 1 {
				correctAnswers = append(correctAnswers, ans.AnswerDesc)
			}
		}

		// 2. 将正确的答案分配到 da_1, da_2, da_3, da_4 指针字段
		// 默认都保持为 nil，对应的就是数据库里的 NULL
		var d1, d2, d3, d4 *string

		// 使用临时变量获取地址，防止迭代循环中的地址复用
		if len(correctAnswers) > 0 {
			val := correctAnswers[0]
			d1 = &val
		}
		if len(correctAnswers) > 1 {
			val := correctAnswers[1]
			d2 = &val
		}
		if len(correctAnswers) > 2 {
			val := correctAnswers[2]
			d3 = &val
		}
		if len(correctAnswers) > 3 {
			val := correctAnswers[3]
			d4 = &val
		}

		// 3. 清洗可能残存的二次转义换行字符
		cleanStem := strings.ReplaceAll(item.Stem, `\n`, "\n")
		cleanStem = strings.ReplaceAll(cleanStem, `\r\n`, "\n")

		// 4. 构建多选题模型
		row := Duoxuan{
			ID:  item.QueId,
			Nr:  cleanStem,
			Da1: d1,
			Da2: d2,
			Da3: d3,
			Da4: d4,
		}
		rowsToInsert = append(rowsToInsert, row)
	}

	// 执行批量存入
	if len(rowsToInsert) > 0 {
		// db.Save 在执行具有 NULL 值的更新和插入时也完美支持
		result := db.Save(&rowsToInsert)
		if result.Error != nil {
			log.Fatalf("数据保存进数据库失败: %v", result.Error)
		}
		fmt.Printf("🎉 多选题处理成功！已写入/更新 %d 条题目数据（空缺答案已被留空为 NULL）。\n", result.RowsAffected)
	} else {
		fmt.Println("⚠️ 未在接口返回中解析到有效的多选题数据。")
	}
}