update cicd

.drone.yml

@@ -5,13 +5,34 @@ steps:
 # 测试
 - name: code-test
   image: python:3.9
+  volumes:
+    - name: cache
+      path: /opt/web/test
   commands:
   - pip install pytest-html
   - pip install pytest-xdist
   - pip install -r requirements.txt
-  - pytest -v ./ -n 5 --maxfail 10 --pastebin=all --html=./py_test_report.html
-  failure: "fail-fast"
-  
+  - pytest -v ./ --maxfail 10 --pastebin=all --html=/opt/web/test/py_test_report.html
+  #- pytest -v ./ -n 5 --maxfail 10 --pastebin=all --html=/opt/web/test/py_test_report.html
+  #failure: "fail-fast"
+
+- name: dingtalk
+  image: lddsb/drone-dingtalk-message
+  settings:
+    token: 178f267b9b16b168e0a0afb223b3d41f3a58e62180ab5288aa576db02965dd72
+    type: markdown
+    success_color: 008000
+    failure_color: FF0000
+    success_pic: https://img1.baidu.com/it/u=1968518221,4002244040&fm=253&fmt=auto&app=138&f=JPEG?w=640&h=400
+    msg_at_mobiles: 15858193327
+    # https://test.itlym.cn/py_test_report.html
+
+volumes:
+- name: cache
+  host:
+    path: /opt/web/test
+
+
 trigger:
   branch:
   - master    

mockTask/m3u8TsTask.json

@@ -21,6 +21,7 @@
     },
     "charset": null,
     "retryNum": 0,
+    "progressId": "xxx",
     "parent": null,
     "exceptedProxy": null,
     "actuallyProxy": null,
@@ -49,38 +50,6 @@
       "partIndex": 7,
       "filePath": "./test_target/pyTest.ts",
       "duration": "5.000000"
-    },
-    "countDownLatch": {
-      "count": 1081
-    },
-    "progress": {
-      "autoFished": true,
-      "afterFinishCallback": {},
-      "taskId": "xiaoya_bieBiWoDongShou",
-      "startTime": null,
-      "alreadyFinishedAtStart": 0,
-      "stopTime": null,
-      "total": 1081,
-      "processed": 0,
-      "successNum": 0,
-      "failNum": 0,
-      "status": 0,
-      "ext": null,
-      "batchProgress": {
-        "taskId": "xiaoya_bieBiWoDongShou",
-        "startTime": null,
-        "alreadyFinishedAtStart": 0,
-        "stopTime": null,
-        "total": 1081,
-        "processed": 0,
-        "successNum": 0,
-        "failNum": 0,
-        "status": 0,
-        "ext": null
-      }
-    },
-    "tsPartDownloadStatus": {
-      "empty": true
     }
   },
   "spiderResult": null,

requirements.txt

@@ -1,3 +1,6 @@
 beautifulsoup4==4.9.3
+oss2==2.18.1
 pycryptodome==3.18.0
+pytest==7.4.0
 Requests==2.31.0
+pytest-html==3.2.0

spider.py

@@ -6,7 +6,16 @@ import operator
 import base64
 import time
 from Crypto.Cipher import AES
+import oss2
 
+
+
+endpoint = 'http://oss-cn-hongkong.aliyuncs.com'  # Suppose that your bucket is in the Hangzhou region.
+auth = oss2.Auth('LTAI5t5ZyATP7DL4nMdK51eL', 'WnJqm3jpuP0JUMo3to8wzcVHgIrBAB')
+bucket = oss2.Bucket(auth, endpoint, 'oss-aliyun-hk')
+
+
+# 判断流程能否处理
 def support(event):
     return True
 
@@ -76,18 +85,32 @@ def handle(faas__request_wrap):
 
     process_start_time = time.time()
     content = None
+    next_command = None
+
+
     if operator.contains(task['resultType'], '[B'):
         video_bytes = r.content
         if task['tsPart']['key']['method'] != 'NONE':
             key_bytes = base64.b64decode(task['tsPart']['key']['key'])
             iv_bytes = base64.b64decode(task['tsPart']['key']['iv'])
-            video_bytes = Aes_byte(key_bytes, iv_bytes).AES_decrypt(r.content)
-        content = 'save to http://itlym.cn/'
-        file_path = task['tsPart']['filePath']
-        if not os.path.exists(os.path.dirname(file_path)):
-            os.makedirs(os.path.dirname(file_path))
-        with open(file_path, 'wb') as f:
-            f.write(video_bytes)
+            video_bytes = Aes_byte(key_bytes, iv_bytes).AES_decrypt(video_bytes)
+
+        remote_path = 'spider/' + task['progressId'] + '/0000.ts'
+        bucket.put_object(remote_path, video_bytes)
+        content = remote_path
+
+        # file_path = task['tsPart']['filePath']
+        # if not os.path.exists(os.path.dirname(file_path)):
+        #     os.makedirs(os.path.dirname(file_path))
+        # with open(file_path, 'wb') as f:
+        #     f.write(video_bytes)
+        # content = 'file_path'
+
+        next_command = 'upload_to_oss'
+        next_command = 'upload_to_disk'
+        next_command = 'upload_to_ssh'
+        next_command = 'upload_to_webdav'
+        # todo 路径放上下文
 
     else:
         # encoding是从http中的header中的charset字段中提取的编码方式，若header中没有charset字段则默认为ISO - 8859 - 1 编码模式，则无法解析中文，这是乱码的原因
@@ -111,7 +134,11 @@ def handle(faas__request_wrap):
           + ' req_cost=%.2f' % req_cost + ' process_cost=%.2f' % process_cost)
 
     result = {
-        'content': content
+        'process_result': {
+            'spider': context
+        },
+        'content': content,
+        'next-command': next_command
     }
 
-    return result
+    return content

upload_to_webdav.py

@@ -0,0 +1,65 @@
+import handleManager
+import tkinter
+from tkinter import filedialog
+from webdav3.client import Client  # 使用pip install webdavclient3 安装
+
+
+# 加载后
+def after_load(event):
+    return True
+
+
+# 卸载前
+def before_unload(event):
+    return True
+
+
+# 判断流程能否处理
+def support(event):
+    return True
+
+
+webdav_connection_options__jianguo = {
+    'webdav_hostname': "https://dav.jianguoyun.com/dav/",
+    'webdav_login': "cn_lym@foxmail.com",
+    'webdav_password': "atj4hxt8rdeh79aj",
+    'disable_check': True,  # 有的网盘不支持check功能
+}
+    # todo 连接放全局上下文
+client = Client(webdav_connection_options__jianguo)
+upload_target_dir = 'spider_cloud/'
+
+
+def upload(file_path, file_name='____file__name'):
+    if file_name == '____file__name':
+        file_name = file_path.split('/')[-1]  # 获取要上传的文件名
+    # 第一个参数为webdav中的路径，可自定义
+    client.upload(upload_target_dir + file_name, file_path)
+    # 注：若webdav中已经有同名称文件则会进行覆盖
+    return upload_target_dir + file_name
+
+'oss-cn-hongkong.aliyuncs.com'
+def handle(faas__request_wrap):
+
+    # 取出参数和输入上下文
+    params = faas__request_wrap[handleManager.const_key_params]
+    context = faas__request_wrap[handleManager.const_key_context]
+
+    # 拿到任务，调用异步执行，后置操作放结果到 mq
+    if not params['task']:
+        raise RuntimeError('no task define')
+
+    task = params['task']
+
+    headers = task['headers']
+
+
+    result = {
+        'process_result': {
+            'spider': context
+        },
+        'content': content,
+        'next-command': next_command
+    }
+
+    return content