import requests
import os
import urllib
 
class Spider_baidu_image():
    def __init__(self):
        self.url = 'http://image.baidu.com/search/acjson?'
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.\
            3497.81 Safari/537.36'}
        self.headers_image = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.\
            3497.81 Safari/537.36',
            'Referer': 'http://image.baidu.com/search/index?tn=baiduimage&ipn=r&ct=201326592&cl=2&lm=-1&st=-1&fm=result&fr=&sf=1&fmq=1557124645631_R&pv=&ic=&nc=1&z=&hd=1&latest=0&copyright=0&se=1&showtab=0&fb=0&width=&height=&face=0&istype=2&ie=utf-8&sid=&word=%E8%83%A1%E6%AD%8C'}
        self.keyword = input("請(qǐng)輸入搜索圖片關(guān)鍵字:")
        self.paginator = int(input("請(qǐng)輸入搜索頁(yè)數(shù)，每頁(yè)30張圖片："))
 
 
    def get_param(self):
        """
        獲取url請(qǐng)求的參數(shù)，存入列表并返回
        :return:
        """
        keyword = urllib.parse.quote(self.keyword)
        params = []
        for i in range(1, self.paginator + 1):
            params.append(
                'tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord={}&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=&hd=1&latest=0&copyright=0&word={}&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&expermode=&force=&cg=star&pn={}&rn=30&gsm=78&1557125391211='.format(
                    keyword, keyword, 30 * i))
        return params
 
    def get_urls(self, params):
        """
        由url參數(shù)返回各個(gè)url拼接后的響應(yīng)，存入列表并返回
        :return:
        """
        urls = []
        for i in params:
            urls.append(self.url + i)
        return urls
 
    def get_image_url(self, urls):
        image_url = []
        for url in urls:
            json_data = requests.get(url, headers=self.headers).json()
            json_data = json_data.get('data')
            for i in json_data:
                if i:
                    image_url.append(i.get('thumbURL'))
        return image_url
 
    def get_image(self, image_url):
        """
        根據(jù)圖片url，在本地目錄下新建一個(gè)以搜索關(guān)鍵字命名的文件夾，然后將每一個(gè)圖片存入。
        :param image_url:
        :return:
        """
        cwd = os.getcwd()
        file_name = os.path.join(cwd, self.keyword)
        if not os.path.exists(self.keyword):
            os.mkdir(file_name)
        for index, url in enumerate(image_url, start=1):
            with open(file_name + '\\{}.jpg'.format(index), 'wb') as f:
                f.write(requests.get(url, headers=self.headers_image).content)
            if index != 0 and index % 30 == 0:
                print('{}第{}頁(yè)下載完成'.format(self.keyword, index / 30))
 
    def __call__(self, *args, **kwargs):
        params = self.get_param()
        urls = self.get_urls(params)
        image_url = self.get_image_url(urls)
        self.get_image(image_url)
 
if __name__ == '__main__':
    spider = Spider_baidu_image()
    spider()

transforms.Compose([
transforms.CenterCrop(84),
transforms.ToTensor(),
])

data_transforms = {
    'train': transforms.Compose([
        transforms.Resize(84),
        transforms.CenterCrop(84),
        # 轉(zhuǎn)換成tensor向量
        transforms.ToTensor(),
        # 對(duì)圖像進(jìn)行歸一化操作
        # [0.485, 0.456, 0.406]，RGB通道的均值與標(biāo)準(zhǔn)差
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize(84),
        transforms.CenterCrop(84),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}

class my_Data_Set(nn.Module):
    def __init__(self, txt, transform=None, target_transform=None, loader=None):
        super(my_Data_Set, self).__init__()
        # 打開存儲(chǔ)圖像名與標(biāo)簽的txt文件
        fp = open(txt, 'r')
        images = []
        labels = []
        # 將圖像名和圖像標(biāo)簽對(duì)應(yīng)存儲(chǔ)起來
        for line in fp:
            line.strip('\n')
            line.rstrip()
            information = line.split()
            images.append(information[0])
            labels.append(int(information[1]))
        self.images = images
        self.labels = labels
        self.transform = transform
        self.target_transform = target_transform
        self.loader = loader
 
    # 重寫這個(gè)函數(shù)用來進(jìn)行圖像數(shù)據(jù)的讀取
    def __getitem__(self, item):
        # 獲取圖像名和標(biāo)簽
        imageName = self.images[item]
        label = self.labels[item]
        # 讀入圖像信息
        image = self.loader(imageName)
        # 處理圖像數(shù)據(jù)
        if self.transform is not None:
            image = self.transform(image)
        return image, label
 
    # 重寫這個(gè)函數(shù)，來看數(shù)據(jù)集中含有多少數(shù)據(jù)
    def __len__(self):
        return len(self.images)
 
# 生成Pytorch所需的DataLoader數(shù)據(jù)輸入格式
train_dataset = my_Data_Set('train.txt', transform=data_transforms['train'], loader=Load_Image_Information)
test_dataset = my_Data_Set('val.txt', transform=data_transforms['val'], loader=Load_Image_Information)
train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=10, shuffle=True)

# 驗(yàn)證是否生成DataLoader格式數(shù)據(jù)
for data in train_loader:
    inputs, labels = data
    print(inputs)
    print(labels)
for data in test_loader:
    inputs, labels = data
    print(inputs)
    print(labels)

import torch
from torch.utils.data import DataLoader
from torchvision import transforms, datasets
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import Dataset
import numpy as np
import os
from PIL import Image
import warnings
import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
 
plt.ion()

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
 
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 18 * 18, 800)
        self.fc2 = nn.Linear(800, 120)
        self.fc3 = nn.Linear(120, 10)
 
    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 18 * 18)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
 
        return x
 
net = Net()

#訓(xùn)練
cirterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.5)
for epoch in range(50):
    running_loss = 0.0
    for i, data in enumerate(train_loader, 0):
        inputs, labels = data
        inputs, labels = Variable(inputs), Variable(labels)
        optimizer.zero_grad()                        # 優(yōu)化器清零
        outputs = net(inputs)
        loss = cirterion(outputs, labels)
        loss.backward()
        optimizer.step()                         #優(yōu)化
        running_loss += loss.item()
        if i % 200 == 199:
            print('[%d %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 200))
            running_loss = 0.0
 
print('finished training!')

#測(cè)試
correct = 0
total = 0
with torch.no_grad():
    for data in test_loader:
        images, labels = data
        outputs = net(Variable(images))
        _, predicted = torch.max(outputs.data, dim=1)
        total += labels.size(0)
        correct += (predicted == labels).sum()
print('Accuracy of the network on the 400 test images: %d %%' % (100 * correct / total))

import torch
from torch.utils.data import DataLoader
from torchvision import transforms, datasets
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torch.autograd import Variable
from torch.utils.data import Dataset
import numpy as np
import os
from PIL import Image
import warnings
import matplotlib.pyplot as plt
warnings.filterwarnings("ignore")
 
plt.ion()
 
 
 
data_transforms = {
    'train': transforms.Compose([
        transforms.Resize(84),
        transforms.CenterCrop(84),
        # 轉(zhuǎn)換成tensor向量
        transforms.ToTensor(),
        # 對(duì)圖像進(jìn)行歸一化操作
        # [0.485, 0.456, 0.406]，RGB通道的均值與標(biāo)準(zhǔn)差
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize(84),
        transforms.CenterCrop(84),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}
def Load_Image_Information(path):
    # 圖像存儲(chǔ)路徑
    image_Root_Dir= r'C:/Users/wbl/Desktop/pythonProject1/image/'
    # 獲取圖像的路徑
    iamge_Dir = os.path.join(image_Root_Dir, path)
    # 以RGB格式打開圖像
    # Pytorch DataLoader就是使用PIL所讀取的圖像格式
    return Image.open(iamge_Dir).convert('RGB')
 
 
class my_Data_Set(nn.Module):
    def __init__(self, txt, transform=None, target_transform=None, loader=None):
        super(my_Data_Set, self).__init__()
        # 打開存儲(chǔ)圖像名與標(biāo)簽的txt文件
        fp = open(txt, 'r')
        images = []
        labels = []
        # 將圖像名和圖像標(biāo)簽對(duì)應(yīng)存儲(chǔ)起來
        for line in fp:
            line.strip('\n')
            line.rstrip()
            information = line.split()
            images.append(information[0])
            labels.append(int(information[1]))
        self.images = images
        self.labels = labels
        self.transform = transform
        self.target_transform = target_transform
        self.loader = loader
 
    # 重寫這個(gè)函數(shù)用來進(jìn)行圖像數(shù)據(jù)的讀取
    def __getitem__(self, item):
        # 獲取圖像名和標(biāo)簽
        imageName = self.images[item]
        label = self.labels[item]
        # 讀入圖像信息
        image = self.loader(imageName)
        # 處理圖像數(shù)據(jù)
        if self.transform is not None:
            image = self.transform(image)
        return image, label
 
    # 重寫這個(gè)函數(shù)，來看數(shù)據(jù)集中含有多少數(shù)據(jù)
    def __len__(self):
        return len(self.images)
 
 
# 生成Pytorch所需的DataLoader數(shù)據(jù)輸入格式
train_dataset = my_Data_Set('train.txt', transform=data_transforms['train'], loader=Load_Image_Information)
test_dataset = my_Data_Set('val.txt', transform=data_transforms['val'], loader=Load_Image_Information)
train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True)
test_loader = DataLoader(test_dataset, batch_size=10, shuffle=True)
 
'''
# 驗(yàn)證是否生成DataLoader格式數(shù)據(jù)
for data in train_loader:
    inputs, labels = data
    print(inputs)
    print(labels)
for data in test_loader:
    inputs, labels = data
    print(inputs)
    print(labels)
'''
 
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
 
        self.conv1 = nn.Conv2d(3, 6, 5)
        self.pool = nn.MaxPool2d(2, 2)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16 * 18 * 18, 800)
        self.fc2 = nn.Linear(800, 120)
        self.fc3 = nn.Linear(120, 10)
 
    def forward(self, x):
        x = self.pool(F.relu(self.conv1(x)))
        x = self.pool(F.relu(self.conv2(x)))
        x = x.view(-1, 16 * 18 * 18)
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
 
        return x
 
net = Net()
 
#訓(xùn)練
cirterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.5)
for epoch in range(50):
    running_loss = 0.0
    for i, data in enumerate(train_loader, 0):
        inputs, labels = data
        inputs, labels = Variable(inputs), Variable(labels)
        optimizer.zero_grad()                        # 優(yōu)化器清零
        outputs = net(inputs)
        loss = cirterion(outputs, labels)
        loss.backward()
        optimizer.step()                         #優(yōu)化
        running_loss += loss.item()
        if i % 200 == 199:
            print('[%d %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 200))
            running_loss = 0.0
 
print('finished training!')
 
#測(cè)試
correct = 0
total = 0
with torch.no_grad():
    for data in test_loader:
        images, labels = data
        outputs = net(Variable(images))
        _, predicted = torch.max(outputs.data, dim=1)
        total += labels.size(0)
        correct += (predicted == labels).sum()
print('Accuracy of the network on the 400 test images: %d %%' % (100 * correct / total))