Skip to content

J
jfV5pc
Commit 3addac14 by 陈玉兰
Options

提交

parent ff6ae920
Showing with 218 additions and 23 deletions
WebContent/WEB-INF/conf/gaowj/bl/business-reptile.xml
......@@ -11,6 +11,7 @@
<value>classpath:com/gaowj/business/fileGrap/fileGrapDAO.xml</value>
<value>classpath:com/gaowj/business/fileDataSort/fileDataSortDAO.xml</value>
<value>classpath:com/gaowj/business/allData/allDataDAO.xml</value>
<value>classpath:com/gaowj/business/textGrapUrl/textGrapUrlDAO.xml</value>
</list>
</property>
</bean>
......@@ -55,4 +56,12 @@
</bean>
</property>
</bean>
<bean id="TextGrapUrlBusiness" class="com.gaowj.business.textGrapUrl.TextGrapUrlBusinessImpl">
<property name="textGrapUrlDAO">
<bean class="org.mybatis.spring.mapper.MapperFactoryBean">
<property name="mapperInterface" value="com.gaowj.business.textGrapUrl.textGrapUrlDAO" />
<property name="sqlSessionFactory" ref="sqlSessionFactory-reptile" />
</bean>
</property>
</bean>
</beans>
WebContent/reptile/fileGrap/fileGrapTab.jsp
......@@ -14,7 +14,7 @@
<table id="list" style="width: 100%; height: 100%;"></table>
<div id="tb">
<a href="#" class="easyui-linkbutton" data-options="iconCls:'icon-add',plain:true" onclick="javascript:addRow()">新增</a>
<a href="#" class="easyui-linkbutton" data-options="iconCls:'icon-reload',plain:true" onclick="javascript:refreshRow()">删除</a>
<a href="#" class="easyui-linkbutton" data-options="iconCls:'icon-reload',plain:true" onclick="javascript:refreshRow()">刷新</a>
<a href="#" class="easyui-linkbutton" data-options="iconCls:'icon-clear',plain:true" onclick="javascript:deleteBatchRow()">删除</a>
</div>
</div>
......
WebContent/reptile/textGrap/js/textGrapTab.js
......@@ -57,8 +57,8 @@ var ls_ywlx = 'YBYW';
//定义grid列表
var ls_title="抓取数据列表";
var is_column = [ [
{field : 'URL',title : '路径',width : 300,align : 'left'},
{field : 'LABEL',title : '标题',width : 100,align : 'left'},
{field : 'TITLE',title : '标题',width : 100,align : 'left'},
{field : 'STATUS',title : '爬取状态',width : 100,align : 'left'},
{field : 'DATABASE_NAME',title : '数据库',width : 100,align : 'left'},
{field : 'TABLE_NAME',title : '表名',width : 100,align : 'left'},
{field : 'CREATE_TIME',title : '创建时间',width : 100,align : 'left'},
......@@ -100,7 +100,8 @@ function formatMenu(value, data,index) {
function getOptionMenu(data,index){
var returnData= "<span class=\"icon-edit\" style=\"padding-left:20px;*width:20px;\"></span><a href=\"javascript:void(0);\" onclick=\"editRow('"
+ index + "')\">编辑</a> | <span class=\"icon-ext-11_10\" style=\"padding-left:20px;*width:20px;\"></span><a href=\"javascript:void(0);\" onclick=\"grapRow('"
+ index + "')\">爬取</a> | <span class=\"icon-search\" style=\"padding-left:20px;*width:20px;\"></span><a href=\"javascript:void(0);\" onclick=\"lookRow('"
+ index + "')\">爬取</a> | <span class=\"icon-ext-13_12\" style=\"padding-left:20px;*width:20px;\"></span><a href=\"javascript:void(0);\" onclick=\"statusRow('"
+ index + "')\">状态</a> | <span class=\"icon-search\" style=\"padding-left:20px;*width:20px;\"></span><a href=\"javascript:void(0);\" onclick=\"lookRow('"
+ index + "')\">查看</a> | <span class=\"icon-clear\" style=\"padding-left:20px;*width:20px;\"></span><a href=\"javascript:void(0);\" onclick=\"deleteRow('"
+ index + "')\">删除</a>";
return returnData;
......@@ -108,7 +109,7 @@ function getOptionMenu(data,index){
//返回菜单集的菜单个数(需更改)
function getOptionMenuCount(){
var is_menu_count = 4;
var is_menu_count = 5;
return is_menu_count;
}
......@@ -132,11 +133,13 @@ function addRow(){
//加载增加子页面成功后的回调
function addUrlOnloadCallback(is_row){
//初始化combobox
init_combobox();
init_combobox_for_datasort();
init_combobox_for_url();
}
function init_combobox(){
//初始化数据分类下拉框
function init_combobox_for_datasort(){
$.ajax({
url:gaowj.WEB_APP_NAME+'/dataSortData_list',
type:'post',
......@@ -150,6 +153,26 @@ function init_combobox(){
})
}
//初始化爬取路径下拉框
function init_combobox_for_url(){
$.ajax({
url:gaowj.WEB_APP_NAME+'/textGrapData_listUrl',
type:'post',
success:function(result){
$('#URL_ID').combobox({
valueField:'UUID',
textField:'NAME',
data:result.rowSet ,
onSelect:function(record){
$("#SAVE_PATH").textbox('setValue',record.SAVE_PATH);
$("#DATABASE_NAME").textbox('setValue',record.DATABASE_NAME);
$("#TABLE_NAME").textbox('setValue',record.TABLE_NAME);
console.log(record);
}
})
}
})
}
//保存
function insertData(is_entity){
var v_result = textGrapModule.textGrapActAction.insertTextGrap(is_entity);
......@@ -178,7 +201,8 @@ function editRow(index){
}
//加载修改子页面成功后的回调
function updateUrlOnloadCallback(is_row){
init_combobox();
init_combobox_for_datasort();
init_combobox_for_url();
}
//保存(is_entity);
function updateData(is_entity){
......@@ -292,11 +316,13 @@ function grapRow(index){
//alert("开发中");
$('#list').datagrid('selectRow',index);
var is_row = $('#list').datagrid('getSelections');
$.ajax({
type:'post',
url:"textGrapAct_grap",
data:{
url:is_row[0].URL,
uuid:is_row[0].UUID,
url_id:is_row[0].URL_ID,
dbName:is_row[0].DATABASE_NAME,
tableName:is_row[0].TABLE_NAME
},
......@@ -314,3 +340,26 @@ function lookRow(index){
$('#list').datagrid('clearSelections');
window.location = 'textGrapView_textGrapDataTab?tablename='+is_row[0].TABLE_NAME;
}
function statusRow(index){
$('#list').datagrid('selectRow',index);
var is_row = $('#list').datagrid('getSelections');
var map={};
map.FIELD_UUID=is_row[0].UUID;
/* map.FIELD_CREATE_ID=is_row[0].CREATE_ID
map.FIELD_CREATE_TIME=is_row[0].CREATE_TIME
map.FIELD_DATABASE_NAME=is_row[0].DATABASE_NAME
map.FIELD_DATA_SORT_ID=is_row[0].DATA_SORT_ID
map.FIELD_SAVE_PATH=is_row[0].SAVE_PATH
map.FIELD_TABLE_NAME=is_row[0].TABLE_NAME*/
map.FIELD_URL_ID=is_row[0].URL_ID;
map.FIELD_TASK_ID=is_row[0].TASK_ID;
$.ajax({
type:'post',
url:"textGrapAct_status",
data:map,
success:function(data){
$('#list').datagrid('reload');
}
});
}
\ No newline at end of file
WebContent/reptile/textGrap/textGrapTabDetail.jsp
......@@ -8,7 +8,7 @@
<span>URL:</span>
</td>
<td style="width: 100%">
<input class="easyui-textbox" id="URL" name="FIELD_URL" style="width:300px" value="${is_row.URL}" required="required"/>
<input class="easyui-combobox" id="URL_ID" name="FIELD_URL_ID" style="width:300px" value="${is_row.URL_ID}" required="required"/>
</td>
</tr>
<tr>
......@@ -22,22 +22,22 @@
<tr>
<td style="width: 20%;" class="tableStyleLable">
<span>
label:
标题:
</span>
</td>
<td style="width: 100%">
<input class="easyui-textbox" id="LABEL" name="FIELD_LABEL" style="width:300px" value="${is_row.LABEL}"/>
<input class="easyui-textbox" id="TITLE" name="FIELD_TITLE" style="width:300px" value="${is_row.TITLE}"/>
</td>
</tr>
<tr>
<td style="width: 20%;" class="tableStyleLable">
<span>
xpath:
存放路径:
</span>
</td>
<td style="width: 100%">
<input class="easyui-textbox" id="XPATH" name="FIELD_XPATH" style="width:300px" value="${is_row.XPATH}"/>
<input class="easyui-textbox" id="SAVE_PATH" name="FIELD_SAVE_PATH" style="width:300px" value="${is_row.SAVE_PATH}"/>
</td>
</tr>
<tr>
......
src_reptile/com/gaowj/business/TextGrapUrlBusiness.java 0 → 100644
package com.gaowj.business;
import java.util.List;
import java.util.Map;
import org.springframework.stereotype.Service;
import com.gaowj.business.exception.BusinessException;
@Service
public interface TextGrapUrlBusiness {
List<Map<String, Object>> list_textgrap_url(Map<String,Object> entity) throws BusinessException ;
}
src_reptile/com/gaowj/business/textGrapUrl/TextGrapUrlBusinessImpl.java 0 → 100644
package com.gaowj.business.textGrapUrl;
import java.util.List;
import java.util.Map;
import com.gaowj.business.TextGrapUrlBusiness;
import com.gaowj.business.exception.BusinessException;
public class TextGrapUrlBusinessImpl implements TextGrapUrlBusiness {
private textGrapUrlDAO textGrapUrlDAO;
public textGrapUrlDAO getTextGrapUrlDAO() {
return textGrapUrlDAO;
}
public void setTextGrapUrlDAO(textGrapUrlDAO textGrapUrlDAO) {
this.textGrapUrlDAO = textGrapUrlDAO;
}
@Override
public List<Map<String, Object>> list_textgrap_url(
Map<String, Object> entity) throws BusinessException {
// TODO Auto-generated method stub
return textGrapUrlDAO.list_textgrap_url(entity);
}
}
src_reptile/com/gaowj/business/textGrapUrl/textGrapUrlDAO.java 0 → 100644
package com.gaowj.business.textGrapUrl;
import java.util.List;
import java.util.Map;
import com.gaowj.business.exception.BusinessException;
public interface textGrapUrlDAO {
List<Map<String, Object>> list_textgrap_url(Map<String,Object> entity) throws BusinessException ;
}
src_reptile/com/gaowj/business/textGrapUrl/textGrapUrlDAO.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.gaowj.business.textGrapUrl.textGrapUrlDAO">
<select id="list_textgrap_url" parameterType="map" resultType="upperCaseKeyMap">
<![CDATA[
select f.*
from C_T_REPTILE_TEXTGRAP_URL f where 1=1
]]>
<if test="UUID != null and UUID != ''">
and f.UUID = #{UUID}
</if>
order by f.CREATE_TIME DESC
</select>
<select id="listCount_textgrap_url" parameterType="map" resultType="int">
<![CDATA[
select count(*) c from C_T_REPTILE_TEXTGRAP_URL f where 1=1
]]>
<if test="UUID != null and UUID != ''">
and f.UUID = #{UUID}
</if>
</select>
</mapper>
\ No newline at end of file
src_reptile/com/gaowj/textGrap/actAction.java
......@@ -6,8 +6,11 @@ import java.util.Map;
import net.sf.json.JSONObject;
import com.gaowj.business.AllDataBusiness;
import com.gaowj.business.FileGrapBusiness;
import com.gaowj.business.SystemOpeBusiness;
import com.gaowj.business.TextGrapBusiness;
import com.gaowj.business.TextGrapUrlBusiness;
import com.gaowj.business.action.BasicAction;
import com.gaowj.business.comp.BusinessManager;
import com.gaowj.business.exception.BusinessException;
......@@ -54,9 +57,6 @@ public class actAction extends BasicAction{
SystemOpeBusiness sysbusiness = BusinessManager.getBusiness(SystemOpeBusiness.class);
sysbusiness.insert_logBusiness("insertTextGrap", "" + entity,
"增加爬虫文本爬取数据", (String) entity.get("UUID"), (String) entity.get("NAME"), "reptile_textgrap");
//
//更新缓存
//CacheKit.update("ehcache11", "com_gaowj_datasort_dataAction_list", business.list_textGrap(null);
jsonObject.putAll(Status.getStatusSuccessMessage("保存成功"));
} catch (Exception e) {
jsonObject.putAll(Status.getStatusErrorMessage("保存失败"));
......@@ -111,21 +111,59 @@ public class actAction extends BasicAction{
}
public String grap() throws BusinessException{
Map<String, Object> entity = RequestUtils.getUpdateFieldMap(request,RequestUtils.UPDATEDATAFIELDPREFIX);
String url=RequestUtils.getString(request, "url");
TextGrapBusiness grapbusiness = BusinessManager.getBusiness(TextGrapBusiness.class);
TextGrapUrlBusiness business = BusinessManager.getBusiness(TextGrapUrlBusiness.class);
String url_id=RequestUtils.getString(request, "url_id");
String textgrap_id=RequestUtils.getString(request, "uuid");
String dbName=RequestUtils.getString(request, "dbName");
String tableName=RequestUtils.getString(request, "tableName");
Map<String, Object> param=new HashMap<String, Object>();
param.put("dbname", dbName);
param.put("tablename", tableName);
com.alibaba.fastjson.JSONObject jsObject=new com.alibaba.fastjson.JSONObject(param);
Map<String, Object> param_grapurl=new HashMap<String, Object>();
param_grapurl.put("UUID", url_id);
List<Map<String, Object>> result=business.list_textgrap_url(param_grapurl);
Map<String, Object> map=result.get(0);
String url=null;
if(map.get("URL")!=null&&""!=map.get("URL")){
url=map.get("URL").toString();
}
Map<String, Object> param_grap=new HashMap<String, Object>();
param_grap.put("dbname", dbName);
param_grap.put("tablename", tableName);
com.alibaba.fastjson.JSONObject jsObject=new com.alibaba.fastjson.JSONObject(param_grap);
String json=jsObject.toJSONString();
ResponseBack responseBack=RestUtil.post(url,json, "utf-8");
Map<String, Object> mapforup=new HashMap<String, Object>();
if(responseBack.getStatusCode()==200){
String taskid=responseBack.getResponseBody();
mapforup.put("UUID", textgrap_id);
mapforup.put("TASK_ID", taskid);
grapbusiness.update_textGrap(mapforup);
}
jsonObject.putAll(Status.getStatusSuccessMessage("开始抓取"));
jsonObject.put("statusCode", responseBack.getStatusCode());
jsonObject.put("responseBody", responseBack.getResponseBody());
return "json";
}
public String status()throws BusinessException{
TextGrapBusiness business = BusinessManager.getBusiness(TextGrapBusiness.class);
TextGrapUrlBusiness urlbusiness = BusinessManager.getBusiness(TextGrapUrlBusiness.class);
Map<String, Object> entity = RequestUtils.getUpdateFieldMap(request,RequestUtils.UPDATEDATAFIELDPREFIX);
Map<String, Object> param=new HashMap<String, Object>();
param.put("UUID", entity.get("URL_ID"));
List<Map<String, Object>> result=urlbusiness.list_textgrap_url(param);
Map<String, Object> map=result.get(0);
String url=map.get("GET_STATUS_URL").toString();
Map<String, Object> param_post=new HashMap<String, Object>();
param_post.put("task_id", entity.get("TASK_ID"));
com.alibaba.fastjson.JSONObject jsObject=new com.alibaba.fastjson.JSONObject(param_post);
String json=jsObject.toJSONString();
ResponseBack responseBack=RestUtil.post(url,json, "utf-8");
if(responseBack.getStatusCode()==200){
entity.put("STATUS", responseBack.getResponseBody());
business.update_textGrap(entity);
}
return "json";
}
}
src_reptile/com/gaowj/textGrap/dataAction.java
......@@ -3,6 +3,7 @@ package com.gaowj.textGrap;
import java.sql.SQLException;
import java.text.ParseException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import net.sf.json.JSONObject;
......@@ -10,6 +11,7 @@ import net.sf.json.JSONObject;
import com.gaowj.business.AllDataBusiness;
import com.gaowj.business.DataSortBusiness;
import com.gaowj.business.TextGrapBusiness;
import com.gaowj.business.TextGrapUrlBusiness;
import com.gaowj.business.action.BasicAction;
import com.gaowj.business.comp.BusinessManager;
import com.gaowj.business.exception.BusinessException;
......@@ -122,4 +124,16 @@ public class dataAction extends BasicAction {
}
public String listUrl() throws BusinessException {
TextGrapUrlBusiness business = BusinessManager.getBusiness(TextGrapUrlBusiness.class);
Map<String, Object> query = null;
List<Map<String, Object>> pageRows = business.list_textgrap_url(query);
Map<String, Object> data = new HashMap<String, Object>();
data.put("rowSet", pageRows);
data.put("rows", data.get("rowSet"));
jsonObject = new JSONObject();
jsonObject.putAll(data);
return "json";
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment