es同步和田玉

ch-admin-api/pom.xml

@@ -120,6 +120,10 @@
             <artifactId>jackson-annotations</artifactId>
             <version>2.12.4</version>
         </dependency>
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-test</artifactId>
+        </dependency>
 
     </dependencies>
 

ch-admin-api/src/main/java/com/wwdz/ch/admin/controller/ItemController.java

@@ -109,8 +109,10 @@ public class ItemController {
     }
 
 
-//    @GetMapping("/synToEs")
+    @GetMapping("/synToEs")
     public void synToEs() {
         synCoinJob.synItemToEs();
     }
+
+
 }

ch-admin-api/src/main/java/com/wwdz/ch/admin/job/ImportNephriteDataJob.java

+package com.wwdz.ch.admin.job;
+
+import com.alibaba.excel.EasyExcel;
+import com.alibaba.excel.EasyExcelFactory;
+import com.alibaba.excel.ExcelReader;
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+import com.alibaba.excel.read.listener.ReadListener;
+import com.alibaba.excel.read.metadata.ReadSheet;
+import com.wwdz.ch.core.consts.CommonEnum;
+import com.wwdz.ch.core.consts.ItemKindEnum;
+import com.wwdz.ch.core.consts.ItemStateEnum;
+import com.wwdz.ch.db.dao.ItemDao;
+import com.wwdz.ch.db.domain.Item;
+import com.xxdxxs.utils.DateUtils;
+import com.xxdxxs.utils.ExcelRead;
+import com.xxdxxs.utils.JsonUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.data.repository.init.ResourceReader;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
+import org.springframework.web.multipart.MultipartFile;
+
+import java.io.*;
+import java.text.DecimalFormat;
+import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.stream.IntStream;
+
+@Component
+public class ImportNephriteDataJob {
+
+    private static final Logger logger = LoggerFactory.getLogger(ImportNephriteDataJob.class);
+
+    private final static String FILE_NAME = "nephrite.xlsx";
+
+    private final static String FILE_PATH = "/nephrite.xlsx";
+
+    private final static String PRE_URL = "https://cdn.wanwudezhi.com";
+
+    ExecutorService executorService = Executors.newFixedThreadPool(10);
+
+    @Autowired
+    ItemDao itemDao;
+
+    public void execute() {
+        logger.info("-------------------- 开始执行, 导入和田玉excel数据 -------------------- ");
+        try {
+            InputStream inputStream = this.getClass().getClassLoader().getResourceAsStream(FILE_NAME);
+            StringArrayExcelReadListener listener = new StringArrayExcelReadListener();
+            ExcelReader reader = EasyExcelFactory.read(inputStream, listener).build();
+            ReadSheet readSheet = EasyExcel.readSheet(0).build();
+            reader.read(readSheet);
+            reader.finish();
+            List<String> head = listener.getHead();
+            logger.info("excel title = {}", head);
+            List<List<String>> dataList = listener.getData();
+            logger.info(">>>>>>>>>> excel 读取数据 : {} 条", dataList.size());
+            Date time = DateUtils.parseString("2022-10-01 00:00:00");
+
+            dataList.parallelStream().forEach(array -> {
+                String oldId = array.get(0);
+                try {
+                    String bidTime = array.get(1);
+                    long price = StringUtils.isEmpty(array.get(4)) ? 0 : Math.round((Double.valueOf(array.get(4))) * 100);
+                    String name = array.get(5);
+                    String videos = array.get(6);
+                    if (StringUtils.hasLength(videos)) {
+                        List<String> newVideoList = new ArrayList<>();
+                        List<String> videoList = Arrays.asList(videos.split(";"));
+                        StringBuffer stringBuffer = new StringBuffer();
+                        videoList.forEach(v -> {
+                            String video = PRE_URL + v;
+                            newVideoList.add(video);
+                        });
+                        videos = String.join(";", newVideoList);
+                    }
+
+                    String images = array.get(7);
+                    if (StringUtils.hasLength(images)) {
+                        List<String> newImageList = new ArrayList<>();
+                        List<String> imageList = Arrays.asList(images.split(";"));
+                        StringBuffer stringBuffer = new StringBuffer();
+                        imageList.forEach(i -> {
+                            String image = PRE_URL + i;
+                            newImageList.add(image);
+                        });
+                        images = String.join(";", newImageList);
+                    }
+                    String detail = null;
+                    try {
+                        detail = array.get(8);
+                    } catch (Exception e) {
+
+                    }
+
+                    Map<String, Object> arrMap = new HashMap<>();
+                    IntStream.rangeClosed(9, 18).forEach(i -> {
+                        try {
+                            String attribute = array.get(i);
+                            if (StringUtils.hasLength(attribute)) {
+                                String[] arr = attribute.split(":");
+                                if (arr.length > 1) ;
+                                arrMap.put(arr[0], arr[1]);
+                            //    logger.info("id :{}, array[{}]: {}", oldId, i, attribute);
+                            }
+                        } catch (Exception e) {
+                        }
+                    });
+                    arrMap.put("oldId", oldId);
+                    String extra = JsonUtils.fromMap(arrMap);
+                //    logger.info("id : {}, extra : {}", oldId, extra);
+                    Item item = new Item();
+                    item.setName(name);
+                    item.setCid(1009596);
+                    item.setExtra(extra);
+                    item.setIsOnSale(false);
+                    item.setIsStd(false);
+                    item.setSort(1);
+                    item.setImages(images);
+                    item.setVideos(videos);
+                    item.setShareImage(null);
+                    item.setTopImage(null);
+                    item.setPrice(price);
+                    item.setDetail(detail);
+                    item.setCreateTime(time);
+                    item.setSourceType(CommonEnum.SourceTypeEnum.SYSTEM_GET.getCode());
+                    item.setUpdateTime(time);
+                    item.setIsDeleted(false);
+                    item.setBidTime(bidTime);
+                    item.setRemark(null);
+                    item.setState(ItemStateEnum.PENDING.getCode());
+                    item.setOriginalSourceUrl(null);
+                    item.setOriginalExtra(null);
+                    item.setSource(CommonEnum.SourceEnum.OTHER_CHANNEL.getCode());
+                    item.setKind(ItemKindEnum.NEPHRITE.getCode());
+                    logger.info(" >>>>> item = {}", item);
+                    itemDao.insert(item);
+                    logger.info(">>>> id : {}, insert success ！<<<<", oldId);
+                } catch (Exception e) {
+                    logger.error("导入excel数据id :{}, error:{}", oldId, e);
+                }
+            });
+            logger.info(">>>>>>>>>>>>>>>>>>>>> 导入和田玉excel数据完成 <<<<<<<<<<<<<<<<<<<");
+        } catch (Exception e) {
+            logger.error("导入excel数据出错 :{}", e);
+        }
+
+    }
+
+   /* public static void main(String[] args) throws IOException {
+        InputStream inputStream = ImportNephriteDataJob.class.getClassLoader().getResourceAsStream(FILE_NAME);
+        MultipartFile multipartFile = new MockMultipartFile(FILE_NAME, FILE_NAME, ".xlsx", inputStream);
+        System.out.println(multipartFile.getOriginalFilename());
+        excelImport();
+    }*/
+
+
+    public static void excelImport() throws IOException {
+        InputStream inputStream = ImportNephriteDataJob.class.getClassLoader().getResourceAsStream(FILE_NAME);
+    //    MultipartFile multipartFile = new MockMultipartFile(FILE_NAME, FILE_NAME, ".xlsx", inputStream);
+        StringArrayExcelReadListener listener = new StringArrayExcelReadListener();
+        ExcelReader reader = EasyExcelFactory.read(inputStream, listener).build();
+        // 读取Sheet,从第0行开始读取(表示从表头开始读)
+        ReadSheet readSheet = EasyExcel.readSheet(0).build();
+        reader.read(readSheet);
+        reader.finish();
+        List<String> head = listener.getHead();
+        System.out.println(JsonUtils.from(head));
+        List<List<String>> data = listener.getData();
+        System.out.println(JsonUtils.from(data));
+    }
+
+
+    public static void main(String[] args) {
+        String a = "1304";
+        long price = StringUtils.isEmpty(a) ? 0 : Math.round((Double.valueOf(a)) * 100);
+        System.out.println(price);
+    }
+}

ch-admin-api/src/main/java/com/wwdz/ch/admin/job/StringArrayExcelReadListener.java

+package com.wwdz.ch.admin.job;
+
+import com.alibaba.excel.context.AnalysisContext;
+import com.alibaba.excel.event.AnalysisEventListener;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class StringArrayExcelReadListener extends AnalysisEventListener<Map<String, String>> {
+
+    /**
+     * 存储读取到的表头
+     */
+    private List<String> head = new ArrayList<>();
+    /**
+     * 存储读取到的 Excel 数据
+     */
+    private List<List<String>> data = new ArrayList<>();
+
+    /**
+     * 每解析一行都会回调invoke()方法
+     * @param item  读取后的数据对象
+     * @param context 内容
+     */
+    @Override
+    public void invoke(Map<String, String> item, AnalysisContext context) {
+        if(item != null && !item.isEmpty()) {
+            List<String> info = item.entrySet().stream().map(e -> e.getValue()).collect(Collectors.toList());
+            data.add(info);
+        }
+    }
+
+    @Override
+    public void doAfterAllAnalysed(AnalysisContext context) {
+
+    }
+
+    /**
+     * 处理读取到的表头数据
+     * @param headMap
+     * @param context
+     */
+    @Override
+    public void invokeHeadMap(Map<Integer, String> headMap, AnalysisContext context) {
+        if(headMap != null && !headMap.isEmpty()) {
+            head = headMap.entrySet().stream().map(e -> e.getValue()).collect(Collectors.toList());
+        }
+    }
+
+    /**
+     * 获取表头数据信息
+     * @return
+     */
+    public List<String> getHead() {
+        return this.head;
+    }
+
+    /**
+     * 获取读取到的 Excel 数据
+     * @return
+     */
+    public List<List<String>> getData() {
+        return this.data;
+    }
+}

ch-admin-api/src/main/java/com/wwdz/ch/admin/job/SynCoinJob.java

 package com.wwdz.ch.admin.job;
 
 import com.github.pagehelper.PageInfo;
-import com.wwdz.ch.core.consts.CommConsts;
-import com.wwdz.ch.core.consts.CommonEnum;
-import com.wwdz.ch.core.consts.ItemStateEnum;
-import com.wwdz.ch.core.consts.MediaTypeEnum;
+import com.wwdz.ch.core.consts.*;
 import com.wwdz.ch.core.storage.QiniuStorage;
 import com.wwdz.ch.core.util.RedisUtils;
 import com.wwdz.ch.db.bean.ItemOfEs;
@@ -196,6 +193,7 @@ public class SynCoinJob {
             CoinRequestDto coinRequestDto = new CoinRequestDto();
             coinRequestDto.setPage(1);
             coinRequestDto.setLimit(100);
+            coinRequestDto.setKind(ItemKindEnum.NEPHRITE.getCode());
             PageInfo<Item> pageInfo = itemDao.findPageExcludeFilter(coinRequestDto);
             int totalPageNum = pageInfo.getPages();
             logger.info("total page : {}, total : {}", totalPageNum, pageInfo.getTotal());

ch-admin-api/src/test/java/com/wwdz/ch/admin/impl/SynCoinJobTest.java

 package com.wwdz.ch.admin.impl;
 
+import com.wwdz.ch.admin.job.ImportNephriteDataJob;
 import com.wwdz.ch.admin.job.SynCoinJob;
 import com.wwdz.ch.admin.job.SynItemToEsJob;
 import com.wwdz.ch.db.dao.ItemDao;
@@ -11,9 +12,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
 
-import java.util.Date;
-
-import static org.junit.Assert.*;
 @SpringBootTest
 @RunWith(SpringJUnit4ClassRunner.class)
 public class SynCoinJobTest {
@@ -30,6 +28,9 @@ public class SynCoinJobTest {
     @Autowired
     ItemDao itemDao;
 
+    @Autowired
+    ImportNephriteDataJob importNephriteDataJob;
+
     @Test
     public void execute() {
         synCoinJob.execute();
@@ -70,4 +71,10 @@ public class SynCoinJobTest {
     public void getMaxTime() throws Exception {
        itemEsDao.getMaxTime();
     }
+
+
+    @Test
+    public void importNephriteData() throws Exception {
+        importNephriteDataJob.execute();
+    }
 }
\ No newline at end of file

ch-core/pom.xml

@@ -118,6 +118,10 @@
                    <artifactId>druid</artifactId>
                    <groupId>com.alibaba</groupId>
                </exclusion>
+               <exclusion>
+                   <artifactId>poi-ooxml</artifactId>
+                   <groupId>org.apache.poi</groupId>
+               </exclusion>
            </exclusions>
        </dependency>
 
@@ -168,6 +172,12 @@
            <version>2.0.23</version>
        </dependency>
 
+       <dependency>
+           <groupId>com.alibaba</groupId>
+           <artifactId>easyexcel</artifactId>
+           <version>2.2.6</version>
+       </dependency>
+
    </dependencies>
     
 </project>

ch-core/src/main/java/com/wwdz/ch/core/consts/CommonEnum.java

@@ -86,7 +86,7 @@ public class CommonEnum {
         CA(10, "CA"),
         YXY(11, "YXY"),
         USER_UPLOAD(1000, "用户上传"),
-
+        OTHER_CHANNEL(2000, "内部其它渠道"),
         ;
 
         private int code;

ch-dao/src/main/java/com/wwdz/ch/db/es/ItemEsDao.java

@@ -36,11 +36,8 @@ public class ItemEsDao {
     @Value("${spring.elasticsearch.rest.index}")
     private String INDEX;
 
-    //商品来源是中国，所对应的编码
-    private final static List<String> CHINESE_SOURCE_LIST = Arrays.asList("1", "2", "3", "7", "11", "1000");
-
-    //来源中国，排除2，7，尚未去除水印的
-    private final static List<String> SOURCE_LIST = Arrays.asList("1", "3", "8",  "11", "1000");
+    //来源中国，排除2，7，尚未去除水印的,2000代表和田玉
+    private final static List<String> SOURCE_LIST = Arrays.asList("1", "3", "8",  "11", "1000", "2000");
 
     @Resource(name="clientByPasswd")
     ElasticsearchClient elasticsearchClient;

ch-dao/src/main/java/com/wwdz/ch/db/impl/ItemDaoImpl.java

@@ -23,7 +23,7 @@ import java.util.stream.IntStream;
 public class ItemDaoImpl implements ItemDao {
 
     //商品来源是中国，所对应的编码
-    private final static List<Integer> CHINESE_SOURCE_LIST = Arrays.asList(1, 2, 3, 7, 8, 10, 11, 1000);
+    private final static List<Integer> CHINESE_SOURCE_LIST = Arrays.asList(1, 2, 3, 7, 8, 10, 11, 1000, 2000);
 
     @Autowired
     ItemMapper itemMapper;
@@ -102,6 +102,8 @@ public class ItemDaoImpl implements ItemDao {
     @Override
     public PageInfo<Item> findPageExcludeFilter(CoinRequestDto coinRequestDto) {
         ItemExample example = new ItemExample();
+        ItemExample.Criteria criteria = example.createCriteria();
+        criteria.andKindEqualTo(coinRequestDto.getKind());
         PageHelper.startPage(coinRequestDto.getPage(), coinRequestDto.getLimit());
         List<Item> list = itemMapper.selectByExampleWithBLOBs(example);
         return new PageInfo<>(list);