from bs4 import BeautifulSoup
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(soup.builder)
    # print(0,soup) #输出整个解析的文档对象(不带格式）
    # print(1,soup.prettify()) #按照格式输出文档内容
    print("- "*30)
    # print(2,soup.div,type(soup.div)) #类型bs4.element.Tag，Tag对象
    # print(3,soup.div["class"]) #会报错，keyError，div没有class属性
    print(3,soup.div.get("class")) #获取div的class属性，没有返回None
    print(4,soup.div.h3["class"]) #多值属性
    print(4,soup.h3.get("class")) #多值属性,获取文档中第一h3标签中的class属性
    print(4,soup.h3.attrs.get("class")) #多值属性
    print(5,soup.img.get("src")) #获取img中src属性值
    soup.img["src"] = "http://www.xddupdate.com" #修改值
    print(5,soup.img["src"])
    print(6,soup.a) #找不到返回None
    del soup.h3["class"] #删除属性
    print(4,soup.h3.get("class"))

print(soup.div.p.string) #第一个div下第一个p的字符串
print(soup.p.string) #同上

from bs4 import BeautifulSoup
from bs4.element import Tag
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(soup.div.string) #返回None，因为多余1个子节点




    

    print("- "*30)
    print("".join(soup.div.strings).strip()) #返回迭代器，带多余的空白字符
    print("- "*30)
    print("".join(soup.div.stripped_strings)) #返回迭代器，去除多余空白字符

from bs4 import BeautifulSoup
from bs4.element import Tag
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(type(soup))
    print(soup.parent)
    print(soup.div.parent.name) #body ,第一个div的父节点
    print(soup.p.parent.parent.get("id")) #取id属性， main
    print("- "*30)
    print(list(map(lambda x:x.name,soup.p.parents))) #父迭代器，由近及远

from bs4 import BeautifulSoup
from bs4.element import Tag
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(type(soup),type(soup.p))
    print("{} [{}]".format(1,soup.p.next_sibling.encode()))
    print("{} [{}]".format(2,soup.p.previous_sibling.encode()))
    print(soup.p.previous_sibling.next_sibling) #等价于soup.p
    print(soup.p.next_sibling.previous_sibling)  # 等价于soup.p
    print(soup.p)
    print(list(soup.p.next_siblings))

from bs4 import BeautifulSoup
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(type(soup),type(soup.p))
    print(soup.p.next_element) #返回"字典"2个字
    print(soup.p.next_element.next_element.encode())
    print(soup.p.next_element.next_element.next_element)
    print(list(soup.p.next_elements))
    print("- "*30)
    #对比差异
    print(list(soup.p.next_elements))
    print(list(soup.p.next_siblings))

import re
print(soup.find_all(re.compile("^h\d"))) #标签名以h开头后接数字

print(soup.find_all(["p","h1","h3"])) #或关系，找出列表所有的标签
print(soup.find_all(re.compile(r"^p|h|\d$"))) #使用正则表达式完成

from bs4 import BeautifulSoup
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(list(map(lambda x: x.name, soup.find_all(True))))
    print(list(map(lambda x: x.name, soup.find_all(None))))
    print(list(map(lambda x: x.name, soup.find_all())))

from bs4 import BeautifulSoup
from bs4.element import Tag
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    values = [True,None,False]
    for value in values:
        all = soup.find_all(value)
        print(type(all[0]))
        print(len(all))
    print("- "*30)
    count = 0
    for i,t in enumerate(soup.descendants): #遍历所有类型的子孙节点
        print(i,type(t),t.name)
        if isinstance(t,Tag): #只对Tag类型计数
            count += 1
    print(count)
# 数目一致，所以返回的是Tag类型的节点，源码中确实返回的Tag类型

from




    
 bs4 import BeautifulSoup
from bs4.element import Tag
def many_classes(tag:Tag):
    # print(type(tag))
    # print(type(tag.attrs))
    return len(tag.attrs.get("class",[])) > 1
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(soup.find_all(many_classes))

print(soup.find_all(class_="content"))
print(soup.find_all(class_="title")) #可以使用任意一个css类
print(soup.find_all(class_="highlight")) #可以使用任意一个css类
print(soup.find_all(class_="highlight title")) #顺序错了，找不到
print(soup.find_all(class_="title highlight")) #顺序一致，找到。就是字符串完全匹配

print(soup.find_all(attrs={"class":"title"}))
print(soup.find_all(attrs={"class":"highlight"}))
print(soup.find_all(attrs={"class":"title highlight"}))
print(soup.find_all(attrs={"id":True}))
print(soup.find_all(attrs={"id":re.compile(r"\d$")}))
print(list(map(lambda x:x.name,soup.find_all(attrs={"id":True,"src":True}))))

from bs4 import BeautifulSoup
import re
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    print(list(map(lambda x:(type(x),x),soup.find_all(text=re.compile("\w+"))))) #返回文本类节点
    print("- "*30)
    print(list(map(lambda x:(type(x),x),soup.find_all(text=re.compile("[a-z]+")))))
    print("- "*30)
    print(soup.find_all(re.compile(r"^(h|p)"),text=re.compile("[a-z]+"))) #相当于过滤Tag对象，并看它的string是否符合text参数要求，返回Tag对象

print(soup.find_all(id=True,limit=3)) #返回列表中有3个结果

from bs4 import BeautifulSoup
with open("d://xdd.html",encoding="utf-8") as f:
    soup =




    
 BeautifulSoup(f,"lxml")
    print(soup.find("img",attrs={"id":"bg1"}).attrs.get("src","xdd"))
    print(soup.find("img",attrs={"id":"bg1"}).get("src")) #简化了attrs
    print(soup.find("img",attrs={"id":"bg1"})["src"])

from bs4 import BeautifulSoup
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    #元素选择器
    print(1,soup.select("p")) #所有的p标签
    #类选择器
    print(2,soup.select(".title"))
    #使用了伪类
    #直接子标签是p的同类型的所有p标签中的第二个
    #(同类型)同标签名p的第2个，伪类只实现了nth-of-type，且要求是数字
    print(3,soup.select("div.content >p:nth-of-type(2)"))
    # id选择器
    print(4,soup.select("p#second"))
    print(5,soup.select("#bg1"))
    #后代选择器
    print(6,soup.select("div p")) # div下逐层找p
    print(7,soup.select("div div p")) #div下逐层找div下逐层找p
    #子选择器，直接后代
    print(8,soup.select("div > p")) #div下直接子标签的p，有2个
    #相邻兄弟选择器
    print(9, soup.select("div p:nth-of-type(1) + [src]")) #返回[]
    print(9, soup.select("div p:nth-of-type(1) + p"))  # 返回p标签
    print(9, soup.select("div > p:nth-of-type(2) + input"))  # 返回input Tag
    print(9, soup.select("div > p:nth-of-type(2) + [type]"))  # 同上
    #普通兄弟选择器
    print(10, soup.select("div p:nth-of-type(1) ~ [src]")) #返回2个img
    #属性选择器
    print(11,soup.select("[src]")) #有属性src
    print(12,soup.select("[src='/']")) #属性src等于/
    print(13,soup.select("[src='http://www.xdd.com/']")) #完全匹配
    print(14,soup.select("[src^='http://www']")) #以http://www开头
    print(15,soup.select("[src$='com/']")) #以com/结尾
    print(16,soup.select("img[src*='xdd']")) #包含xdd
    print(17,soup.select("img[src*='.com']")) #包含.com
    print(18,soup.select("[class='title highlight']")) #完全匹配calss等于'title highlight'
    print(19,soup.select("[class~=title]")) #多值属性中有一个title

from bs4 import BeautifulSoup
with open("d://xdd.html",encoding="utf-8") as f:
    soup = BeautifulSoup(f,"lxml")
    # 元素选择器
    ele = soup.select("div") #所有的div标签
    print(type(ele))
    print(ele[0].string) #内容仅仅只能是文本类型，否则返回None
    print(list(ele[0].strings)) #迭代保留空白字符
    print(list(ele[0].stripped_strings)) #迭代不保留空白字符
    print("- "*30)
    print(ele[0])
    print("- " * 30)
    print(list(ele[0].text))#本质上就是get_text(),保留空白字符的strings
    print(list(ele[0].get_text())) #迭代并join，保留空白字符，strip默认为False
    print(list(ele[0].get_text(strip=True))) #迭代并join，不保留空白字符

class Tag(PageElement):
@property
    def string(self):
        if len(self.contents) != 1:
            return None
        child = self.contents[0]
        if isinstance(child, NavigableString):
            return child
        return child.string
    @string.setter
    def string(self, string):
        self.clear()
        self.append(string.__class__(string))
    def _all_strings(self, strip=False, types=(NavigableString, CData)):
        for descendant in self.descendants:
                (types is None and not isinstance(descendant




    
, NavigableString))
                (types is not None and type(descendant) not in types)):
                continue
            if strip:
                descendant = descendant.strip()
                if len(descendant) == 0:
                    continue
            yield descendant
    strings = property(_all_strings)
    @property
    def stripped_strings(self):
        for string in self._all_strings(True):
            yield string
    def get_text(self, separator="", strip=False,
                 types=(NavigableString, CData)):
        return separator.join([s for s in self._all_strings(
                    strip, types=types)])
    getText = get_text
    text = property(get_text)

from jsonpath import jsonpath
import requests
import json
ua = "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-CN) AppleWebKit/537.36 (KHTML, like Gecko) Version/5.0.1 Safari/537.36"
url = "https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&page_limit=10&page_start=0"
with requests.get(url,headers={"User-agent":ua}) as response:
    if response.status_code==200:
        text = response.text
        print(text[:100])
        js = json.loads(text)
        print(str(js)[:100]) #json转换为Python数据结构
        #知道所有电影的名称
        rs1 = jsonpath(js,"$..title") #从根目录开始，任意层次的title属性
        print(rs1)
        #找到所有subjects
        rs2 = jsonpath(js,"$..subjects")
        print(len(rs2),str(rs2[0])[:100]) #由于太长，取前100个字符
        print("- " * 30)
        # 找到所有得分高于8分的电影名称
        # 根下任意层的subjects的子节点rate大于字符串8
        rs3 = jsonpath(js,'$..subjects[?(@.rate > "8")]') #？()是过滤器
        print(rs3)
        print("- "*30)
        #根下任意层的subjects的子节点rate大于字符串8的节点的子节点title
        rs4 = jsonpath(js,'$..subjects[?(@.rate > "8")].title')
        print(rs4)
        print("- " * 30)
        rs5 = jsonpath(js,"$..subjects[?(@.rate > '6')].title")
        print(rs5[:2])