【python】并发编程

import threading
import requests
import time
# def my_func(a, b):
#     do_craw(a, b)
#
#
# t = threading.Thread(target=my_func, args=(100, 200))
#
# t.start()    # 启动线程
# t.join()    # 等待结束

# 列表解析
urls = [
    f"https://www.cnblogs.com/#p{page}"
    for page in range(1, 50+1)
]


def craw(url):
    r = requests.get(url)
    print(url, len(r.text))


def single_thread():
    print("single thread begin")
    for url in urls:
        craw(url)
    print("single thread end")


def multi_thread():
    print("multi thread begin")
    threads = []
    for url in urls:
        threads.append(
            threading.Thread(target=craw, args=(url, ))
        )
    for thread in threads:
        thread.start()

    for thread in threads:
        thread.join()    # 等待线程结束
    print("multi thread end")


if __name__ == "__main__":
    start = time.time()
    single_thread()
    end = time.time()
    print("single thread cost:", end - start, "seconds")

    start = time.time()
    multi_thread()
    end = time.time()
    print("multi thread cost:", end - start, "seconds")

# 导入类库
import qeue

# 创建Queue
q = queue.Queue()

# 添加元素
q.put(item)   # 阻塞型添加，队列满了，需要等待队列腾出位置再添加

# 获取元素
item = q.get()    # 阻塞型获取，队列空了，需要等待队列中添加了元素再获取

# 查看元素的多少
q.qsize()

# 判断是否为空
q.empty()

# 判断是否已满
q.full()

import threading
import requests
import time
from bs4 import BeautifulSoup


# 列表解析
urls = [
    f"https://www.cnblogs.com/#p{page}"
    for page in range(1, 50+1)
]


def craw(url):
    r = requests.get(url)
    print(url, len(r.text))
    return r.text

  
def parse(html):
    soup = BeautifulSoup(html, "html.parser")
    links = soup.find_all("a", class_="post-item-title")
    return [(link["href"], link.get_text()) for link in links]

import queue
import blog_spider
import time
import random
import threading


def do_craw(url_queue: queue.Queue, html_queue: queue.Queue):
    while True:
        url = url_queue.get()
        html = blog_spider.craw(url)
        html_queue.put(html)
        print(threading.current_thread().name, f"craw {url}",
              "url_queue.size=", url_queue.qsize())
        time.sleep(random.randint(1, 2))


def do_parse(html_queue: queue.Queue, fout):
    while True:
        html = html_queue.get()
        results = blog_spider.parse(html)
        for result in results:
            fout.write(str(result) + "\n")
        print(threading.current_thread().name, f"results.size", len(results),
              "html_queue.size=", html_queue.qsize())
        time.sleep(random.randint(1, 2))


if __name__ == "__main__":
    url_queue = queue.Queue()
    html_queue = queue.Queue()
    for url in blog_spider.urls:
        url_queue.put(url)

    for idx in range(3):
        t = threading.Thread(target=do_craw, args=(url_queue, html_queue),
                             name=f"craw{idx}")
        t.start()

    fout = open("data.txt", "w")
    for idx in range(2):
        t = threading.Thread(target=do_parse(), args=(html_queue, fout),
                             name=f"parse{idx}")
        t.start()

# Lock的两种用法
# 第一种方法
import threading

lock = threading.Lock()

lock.acquire()
try:
  # do something
finally:
  lock.release()
  
 # 第二种方法
import threading

lock = threading.Lock()

with lock:
  # do something

import threading
import time


lock = threading.Lock()


class Account:
    def __init__(self, balance):
        self.balance = balance


def draw(account, amount):
    with lock:
        if account.balance >= amount:
            time.sleep(0.1)    # 切换线程
            print(threading.current_thread().name, "取钱成功")
            account.balance -= amount
            print(threading.current_thread().name, "当前余额", account.balance)
        else:
            print(threading.current_thread().name, "取钱失败，余额不足")


if __name__ == "__main__":
    account = Account(1000)
    ta = threading.Thread(name="ta", target=draw, args=(account, 800))
    tb = threading.Thread(name="tb", target=draw, args=(account, 800))
    ta.start()
    tb.start()
    
    
#  输出结果：
ta 取钱成功
ta 当前余额 200
tb 取钱失败，余额不足

Process finished with exit code 0

# 方法一
from concurrent.futures import ThreadPoolExecutor, as_completed

with ThreadPoolExecutor() as pool:
  results = pool.map(craw, urls)
  for result in results:
    print(result)
    

 # 方法二
with ThreadPoolExecutor() as pool:
  futures = [pool.submit(craw, url) for url in urls]
  for future in futures:
    print(future.result())
  for future in as_completed(futures):
    print(future.result())