2021-07-08
2021/7/9 6:08:03
本文主要是介绍2021-07-08,对大家解决编程问题具有一定的参考价值,需要的程序猿们随着小编来一起学习吧!
import json
import os
from py2neo import Graph, Node, Relationship
class GoodsKg:
def init(self):
cur = ‘/’.join(os.path.abspath(file).split(’/’)[:-1])
self.data_path = os.path.join(cur, ‘data/goods_info.json’)
self.g = Graph(
host=“127.0.0.1”, # neo4j 搭载服务器的ip地址,ifconfig可获取到
http_port=7474, # neo4j 服务器监听的端口号
user=“lhy”, # 数据库user name,如果没有更改过,应该是neo4j
password=“lhy123”)
return
'''读取数据''' def read_data(self): rels_goods = [] rels_brand = [] goods_attrdict = {} concept_goods = set() concept_brand = set() count = 0 for line in open(self.data_path): count += 1 print(count) line = line.strip() data = json.loads(line) first_class = data['fisrt_class'].replace("'",'') second_class = data['second_class'].replace("'",'') third_class = data['third_class'].replace("'",'') attr = data['attrs'] concept_goods.add(first_class) concept_goods.add(second_class) concept_goods.add(third_class) rels_goods.append('@'.join([second_class, 'is_a', '属于', first_class])) rels_goods.append('@'.join([third_class, 'is_a', '属于', second_class])) if attr and '品牌' in attr: brands = attr['品牌'].split(';') for brand in brands: brand = brand.replace("'",'') concept_brand.add(brand) rels_brand.append('@'.join([brand, 'sales', '销售', third_class])) goods_attrdict[third_class] = {name:value for name,value in attr.items() if name != '品牌'} return concept_brand, concept_goods, rels_goods, rels_brand '''构建图谱''' def create_graph(self): concept_brand, concept_goods, rels_goods, rels_brand = self.read_data() # print('creating nodes....') # self.create_node('Product', concept_goods) # self.create_node('Brand', concept_brand) # print('creating edges....') # self.create_edges(rels_goods, 'Product', 'Product') self.create_edges(rels_brand, 'Brand', 'Product') return '''批量建立节点''' def create_node(self, label, nodes): pairs = [] bulk_size = 1000 batch = 0 bulk = 0 batch_all = len(nodes)//bulk_size print(batch_all) for node_name in nodes: sql = """CREATE(:%s {name:'%s'})""" % (label, node_name) pairs.append(sql) bulk += 1 if bulk % bulk_size == 0 or bulk == batch_all+1: sqls = '\n'.join(pairs) self.g.run(sqls) batch += 1 print(batch*bulk_size,'/', len(nodes), 'finished') pairs = [] return '''构造图谱关系边''' def create_edges(self, rels, start_type, end_type): batch = 0 count = 0 for rel in set(rels): count += 1 rel = rel.split('@') start_name = rel[0] end_name = rel[3] rel_type = rel[1] rel_name = rel[2] sql = 'match (m:%s), (n:%s) where m.name = "%s" and n.name = "%s" create (m)-[:%s{name:"%s"}]->(n)' %(start_type, end_type, start_name, end_name,rel_type,rel_name) try: self.g.run(sql) except Exception as e: print(e) if count%10 == 0: print(count) return
if name ==‘main’:
handler = GoodsKg()
handler.create_graph()
#!/usr/bin/env python3
coding: utf-8
File: collect_info.py
Author: lhy<lhy_in_blcu@126.com,https://huangyong.github.io>
Date: 19-3-30
import urllib.request
from urllib.parse import quote_plus
from lxml import etree
import gzip
import chardet
import json
import pymongo
class GoodSchema:
def init(self):
self.conn = pymongo.MongoClient()
return
'''获取搜索页''' def get_html(self, url): headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/600.5.17 (KHTML, like Gecko) Version/8.0.5 Safari/600.5.17"} try: req = urllib.request.Request(url, headers=headers) data = urllib.request.urlopen(req).read() coding = chardet.detect(data) html = data.decode(coding['encoding']) except: req = urllib.request.Request(url, headers=headers) data = urllib.request.urlopen(req).read() html = data.decode('gbk') return html '''获取详情页''' def get_detail_html(self, url): headers = { "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "accept-encoding": "gzip, deflate, br", "accept-language": "en-US,en;q=0.9", "cache-control": "max-age=0", "referer": "https://www.jd.com/allSort.aspx", "upgrade-insecure-requests": 1, "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/66.0.3359.181 Chrome/66.0.3359.181 Safari/537.36" } try: req = urllib.request.Request(url, headers=headers) data = urllib.request.urlopen(req).read() html = gzip.decompress(data) coding = chardet.detect(html) html = html.decode(coding['encoding']) except Exception as e: req = urllib.request.Request(url, headers=headers) data = urllib.request.urlopen(req).read() html = gzip.decompress(data) html = html.decode('gbk') return html '''根据主页获取数据''' def home_list(self): url = 'https://www.jd.com/allSort.aspx' html = self.get_html(url) selector = etree.HTML(html) divs = selector.xpath('//div[@class= "category-item m"]') for indx, div in enumerate(divs): first_name = div.xpath('./div[@class="mt"]/h2/span/text()')[0] second_classes = div.xpath('./div[@class="mc"]/div[@class="items"]/dl') for dl in second_classes: second_name = dl.xpath('./dt/a/text()')[0] third_classes = ['https:' + i for i in dl.xpath('./dd/a/@href')] third_names = dl.xpath('./dd/a/text()') for third_name, url in zip(third_names, third_classes): try: attr_dict = self.parser_goods(url) attr_brand = self.collect_brands(url) attr_dict.update(attr_brand) data = {} data['fisrt_class'] = first_name data['second_class'] = second_name data['third_class'] = third_name data['attrs'] = attr_dict self.conn['goodskg']['data'].insert(data) print(indx, len(divs), first_name, second_name, third_name) except Exception as e: print(e) return '''解析商品数据''' def parser_goods(self, url): html = self.get_detail_html(url) selector = etree.HTML(html) title = selector.xpath('//title/text()') attr_dict = {} other_attrs = ''.join([i for i in html.split('\n') if 'other_exts' in i]) other_attr = other_attrs.split('other_exts =[')[-1].split('];')[0] if other_attr and 'var other_exts ={};' not in other_attr: for attr in other_attr.split('},'): if '}' not in attr: attr = attr + '}' data = json.loads(attr) key = data['name'] value = data['value_name'] attr_dict[key] = value attr_divs = selector.xpath('//div[@class="sl-wrap"]') for div in attr_divs: attr_name = div.xpath('./div[@class="sl-key"]/span/text()')[0].replace(':','') attr_value = ';'.join([i.replace(' ','') for i in div.xpath('./div[@class="sl-value"]/div/ul/li/a/text()')]) attr_dict[attr_name] = attr_value return attr_dict '''解析品牌数据''' def collect_brands(self, url): attr_dict = {} brand_url = url + '&sort=sort_rank_asc&trans=1&md=1&my=list_brand' html = self.get_html(brand_url) if 'html' in html: return attr_dict data = json.loads(html) brands = [] if 'brands' in data and data['brands'] is not None: brands = [i['name'] for i in data['brands']] attr_dict['品牌'] = ';'.join(brands) return attr_dict
if name == ‘main’:
handler = GoodSchema()
handler.home_list()
这篇关于2021-07-08的文章就介绍到这儿,希望我们推荐的文章对大家有所帮助,也希望大家多多支持为之网!
- 2024-12-26JavaScript入门教程:从零开始学习JavaScript编程
- 2024-12-26JavaScript入门教程:从零开始学习JavaScript
- 2024-12-26JS编程入门指南:从零开始学习JavaScript
- 2024-12-25Java编程面试题详解与解答
- 2024-12-25TS基础知识详解:初学者必看教程
- 2024-12-252024面试题解析与攻略:从零开始的面试准备指南
- 2024-12-25数据结构与算法学习:新手入门教程
- 2024-12-25初学者必备:订单系统资料详解与实操教程
- 2024-12-24内网穿透资料入门教程
- 2024-12-24微服务资料入门指南