处理数据导入流程-1

13693261870

2023-08-22 994d438b4b06011eb7656357a367392bb30467c5

处理数据导入流程-1

 DataLoader/CS/DBHelper.cs

@@ -14,20 +14,30 @@
    public class DBHelper
    {
        // id, eventid, metaid, verid, name, type, guid, path, sizes, tab, rows, create_user, create_time, update_user, update_time, bak, geom, layer, depcode, dircode, ismeta, sensortype, acq_time, resolution, gridsize, coor_sys, epsg, h_datum, mata_type, bands, band_type, ct
        public static string insertMeta = "insert into lf.sys_meta (eventid, metaid, verid, name, type, guid, path, sizes, tab, rows, create_user, create_time, bak, geom, layer, depcode, dircode, ismeta, sensortype, acq_time, resolution, gridsize, coor_sys, epsg, h_datum, mata_type, bands, band_type, ct) values (@eventid, @metaid, @verid, @name, @type, @guid, @path, @sizes, @tab, @rows, @create_user, now(), @bak, @geom, @layer, @depcode, @dircode, @ismeta, @sensortype, @acq_time, @resolution, @gridsize, @coor_sys, @epsg, @h_datum, @mata_type, @bands, @band_type, @ct) returning id";
        private const string insertMeta = "insert into lf.sys_meta (eventid, metaid, verid, name, type, guid, path, sizes, tab, rows, create_user, create_time, bak, geom, layer, depcode, dircode, ismeta, sensortype, acq_time, resolution, gridsize, coor_sys, epsg, h_datum, mata_type, bands, band_type, ct) values (@eventid, @metaid, @verid, @name, @type, @guid, @path, @sizes, @tab, @rows, @create_user, now(), @bak, @geom, @layer, @depcode, @dircode, @ismeta, @sensortype, @acq_time, @resolution, @gridsize, @coor_sys, @epsg, @h_datum, @mata_type, @bands, @band_type, @ct) returning id";

        /// <summary>
        /// 插入元数据
        /// </summary>
        public int InsertMeta(List<SysMeta> list)
        public static int InsertMeta(SysMeta meta)
        {
            List<DbParameter> args = Tools.GetParams<SysMeta>(insertMeta, meta);

            return Tools.DBHelper.GetIntScalar(insertMeta, args.ToArray());
        }

        /// <summary>
        /// 插入元数据
        /// </summary>
        public static int InsertMetas(List<SysMeta> list)
        {
            PostgreHelper db = Tools.DBHelper;

            int count = 0;
            foreach (SysMeta meta in list)
            {
                List<DbParameter> args = Tools.GetParams<SysMeta>(insertMeta, meta)
                    ;
                List<DbParameter> args = Tools.GetParams<SysMeta>(insertMeta, meta);

                int id = db.GetIntScalar(insertMeta, args.ToArray());
                if (id > 0) count++;
            }

 DataLoader/CS/Importor.cs

@@ -1,68 +1,68 @@
using DataLoader.Model;
using Npgsql;
using System;
using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Data.Common;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using System.Windows.Forms;

namespace DataLoader.CS
{
    public class Importor
    {
        private static List<String> Exclusions = new List<string>() { 
            "jp2.aux.xml", 
            ".midx", ".strmi", 
            ".rrd", ".img.aux.xml", ".hdr", ".img.enp", ".img.xml", 
            ".tfw", ".tif.ovr", ".tif.aux.xml", ".tif.enp", ".tif.xml", ".prj",
            ".shx", ".dbf", ".cpg" //, ".prj"
        };

        private static PostgreHelper _ph = null;

        public static PostgreHelper Helper
        /// <summary>
        /// 获取文件
        /// </summary>
        public static void GetFiles(ObservableCollection<ViewData> viewDatas)
        {
            get
            string[] files = Directory.GetFiles(CommonProp.SourcePath);
            for (int i = 0, c = files.Length; i < c; i++)
            {
                if (_ph == null) _ph = new PostgreHelper();
                ViewData vd = new ViewData();
                vd.ID = i + 1;
                vd.FilePath = files[i];
                vd.Ext = System.IO.Path.GetExtension(files[i]);
                vd.Status = "准备";
                viewDatas.Add(vd);

                return _ph;
                SetSysMeta(vd);
            }
        }

        private static readonly string insertMeta = "insert into lf.sys_meta (eventid, metaid, dirid, depid, verid, name, type, guid, path, sizes, tab, rows, create_user, create_time) values (@eventid, @metaid, @dirid, @depid, @verid, @name, @type, @guid, @path, @sizes, @tab, @rows, @create_user, now()) returning id";

        public static string GetFilePathByGuid(string guid)
        /// <summary>
        /// 数据导入
        /// </summary>
        public static void ImportFiles(ObservableCollection<ViewData> viewDatas)
        {
            string sql = "select path from lf.sys_meta where guid = @guid limit 1";
            Parallel.ForEach(viewDatas, (vd, ParallelLoopState) =>
            {
                try
                {
                    //

            DbParameter dp = new NpgsqlParameter("@guid", guid);
            object obj = Helper.GetScalar(sql, dp);
                    SysMeta meta = new SysMeta();
                    //

            return obj == null ? null : obj.ToString();
                    int id = DBHelper.InsertMeta(meta);
                    vd.Status = id > 0 ? "成功。" : "失败！";
                }
                catch (Exception ex)
                {
                    LogOut.Error(ex.StackTrace);
                    vd.Status = string.Format("失败：{0}！", ex.Message);
                }
            });
        }

        public static bool IsFileExists(string guid)
        /// <summary>
        /// 复制文件
        /// </summary>
        private static void CopyFile(string source, string target)
        {
            string sql = "select count(*) from lf.sys_meta where guid = @guid";

            DbParameter dp = new NpgsqlParameter("@guid", guid);
            object obj = Helper.GetScalar(sql, dp);

            return obj != null && Convert.ToInt32(obj) > 0;
        }

        public static int InsertMeta(SysMeta meta)
        {
            List<DbParameter> list = Tools.GetParams<SysMeta>(insertMeta, meta);

            object obj = Helper.GetScalar(insertMeta, list.ToArray());

            return obj == null ? 0 : Convert.ToInt32(obj);
            if (File.Exists(source))
            {
                File.Copy(source, target, true);
            }
        }

        public static void GetFiles(ObservableCollection<ViewData> viewDatas, string source)
@@ -97,7 +97,7 @@

        public static void ImportFiles(ObservableCollection<ViewData> viewDatas, string target, int start)
        {
            Parallel.ForEach(viewDatas, (vd, ParallelLoopState) =>
            /*Parallel.ForEach(viewDatas, (vd, ParallelLoopState) =>
            {
                try
                {
@@ -118,19 +118,7 @@

                    vd.Status = "准备入库";
                    vd.Meta = mf;
                }
                catch (Exception ex)
                {
                    LogOut.Error(ex.Message + "\r\n" + ex.StackTrace);
                    vd.Status = "失败！";
                }
            });

            foreach (ViewData vd in viewDatas)
            {
                int metaId = InsertMeta(vd.Meta);
                vd.Status = metaId == 0 ? "元数据出错！" : "完成。";
            }
            });*/
        }

        public static void ImportFiles2(ObservableCollection<ViewData> viewDatas, string target)
@@ -140,19 +128,11 @@
            {
                try
                {
                    vd.Status = "生成MD5码...";
                    string guid = MD5Helper.GetMD5Hash(vd.FilePath);
                    if (IsFileExists(guid))
                    {
                        vd.Status = "已存在！";
                        continue;
                    }

                    vd.Status = "插入数据库...";
                    start = GetSubPath(target, start);
                    SysMeta mf = GetMetaFile(vd, start, guid);
                    SysMeta mf = GetMetaFile(vd, start, "");

                    int metaId = InsertMeta(mf);
                    int metaId = 0; // InsertMeta(mf);
                    if (metaId == 0)
                    {
                        vd.Status = "元数据出错！";
@@ -211,14 +191,6 @@
            mf.create_user = vd.Meta.create_user;

            return mf;
        }

        private static void CopyFile(string source, string target)
        {
            if (File.Exists(source))
            {
                File.Copy(source, target, true);
            }
        }
    }
}

 DataLoader/MainWindow.xaml

@@ -1,7 +1,7 @@
<Window x:Class="DataLoader.MainWindow"
        xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
        xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
        Height="700" Width="900" Loaded="Window_Loaded"
        Height="700" Width="900" Loaded="Window_Loaded" Closing="Window_Closing"
        Title="数据入库 v1.5" WindowStartupLocation="CenterScreen">
    <Window.Resources>
        <Style x:Key="btn" TargetType="Button">

 DataLoader/MainWindow.xaml.cs

@@ -15,6 +15,8 @@
{
    public partial class MainWindow : Window
    {
        private bool isBusy;

        public ResWin resWin;

        private LoginWin loginWin;
@@ -32,8 +34,6 @@
            CommonProp.Owner = this;
            CommonProp.Init();

            //this.cbMetaType.ItemsSource= GetMetaTypes();
            //this.cbMetaType.SelectedIndex= 0;
            this.btnLoad.IsEnabled = false;
            this.btnImport.IsEnabled = false;

@@ -59,7 +59,7 @@
            return list;
        }

        // 设置
        // 登录
        private void Login_MouseLeftButtonDown(object sender, RoutedEventArgs e)
        {
            loginWin = new LoginWin();
@@ -67,6 +67,7 @@
            loginWin.Show();
        }

        // 设置登录信息
        public void SetLoginInfo()
        {
            if (loginWin != null) loginWin.Close();
@@ -88,6 +89,7 @@
            this.resWin.Show();
        }

        // 设置资源信息
        public void SetRes(string dircode, string fullName)
        {
            this.Show();
@@ -126,40 +128,41 @@
        // 加载
        private void Load_MouseLeftButtonDown(object sender, RoutedEventArgs e)
        {
            //if (string.IsNullOrEmpty(SourcePath) || !Directory.Exists(SourcePath))
            //{
            //    MessageBox.Show("请选择数据目录！", "提示");
            //    return;
            //}
            //if (string.IsNullOrEmpty(TargetPath) || !Directory.Exists(TargetPath))
            //{
            //    MessageBox.Show("请选择入库目录！", "提示");
            //    return;
            //}
            if (isBusy) return;
            if (string.IsNullOrEmpty(CommonProp.Dircode))
            {
                MessageBox.Show("请选择资源目录！", "提示");
                return;
            }
            if (string.IsNullOrEmpty(CommonProp.SourcePath) || !Directory.Exists(CommonProp.SourcePath))
            {
                MessageBox.Show("请选择数据目录或数据目录不存在！", "提示");
                return;
            }
            if (string.IsNullOrEmpty(CommonProp.TargetPath) || !Directory.Exists(CommonProp.TargetPath))
            {
                MessageBox.Show("请选择入库目录或入库目录不存在！", "提示");
                return;
            }
            CommonProp.SensorType = this.tbSensorType.Text.Trim();
            ContentControl cc = this.cbMetaType.SelectedItem as ContentControl;
            CommonProp.MetaType = null == cc || "请选择" == cc.Content.ToString() ? string.Empty : cc.Content.ToString();
            CommonProp.AcqTime = this.dpAcqTime.SelectedDate;

            //string[] files = Directory.GetFiles(SourcePath);
            //if (null == files || files.Length == 0)
            //{
            //    MessageBox.Show("数据目录下没有文件！", "提示");
            //    return;
            //}

            //this.viewDatas.Clear();
            //Importor.GetFiles(viewDatas, SourcePath);
            this.isBusy = true;
            this.viewDatas.Clear();
            Importor.GetFiles(viewDatas);
            this.isBusy = false;
        }

        // 导入
        private void Import_MouseLeftButtonDown(object sender, RoutedEventArgs e)
        {
            if (viewDatas.Count == 0) return;
            if (isBusy || viewDatas.Count == 0) return;

            //string str = this.tbStart.Text.Trim();
            //int start = 1;
            //int.TryParse(str, out start);

            this.isBusy = true;
            Thread thread = new Thread(new ThreadStart(() =>
            {
                //Importor.ImportFiles(viewDatas, TargetPath, start);
                Importor.ImportFiles(viewDatas);

                lvView.Dispatcher.BeginInvoke((ThreadStart)delegate
                {
@@ -167,9 +170,21 @@
                    view.Refresh();
                });

                isBusy = false;
                MessageBox.Show("数据导入完成！", "提示");
            }));
            thread.Start();
        }

        // 窗口关闭中事件
        private void Window_Closing(object sender, CancelEventArgs e)
        {
            if (isBusy)
            {
                MessageBox.Show("正在导入数据，请稍后~", "提示");
                e.Cancel = true;
                return;
            }
        }
    }
}

 DataLoader/Model/CommonProp.cs

@@ -76,16 +76,6 @@
        public static string Dirname;

        /// <summary>
        /// 传感器类型
        /// </summary>
        public static string sensortype;

        /// <summary>
        /// 采集时间
        /// </summary>
        public static DateTime acq_time = DateTime.Now;

        /// <summary>
        /// 元数据类型
        /// </summary>
        public static string MetaType;
@@ -108,7 +98,7 @@
        /// <summary>
        /// 采集时间
        /// </summary>
        public static DateTime AcqTime;
        public static DateTime? AcqTime;

        /*private string _sourcePath;