Archive for May, 2008

Restoring a backup. Wait, where are my permissions? or “How to replace all permissions in one directory tree with the permissions from another identical tree”

Ok, I’m starting to get fond of these “a slice of reality” or “what am I going to show you” title blog posts…

So once again this piece of code is a bit drastic. It’s neatly written if you ask me and should show you how you can replace the file security settings (ACLs) with some other files security settings (same for directories). The only meaningfull place to do this (off the top of my head) is when you have two identical files and for some reason the one file has the correct permissions and the other doesn’t. But daily developer business is always full of these little request, and it’s like a personal search for the holy grail - you just have to give it a try.

So if you ever want to copy the exact permission structure from one directory tree to another (for example you used some copy tool that doesn’t copy the permissions) then you can use this little tool to do just that.

A word of warning (as always). Have a backup of your permissions (most probably you will, because you need to copy them from somewhere) and of course run this tool at your own risk. The binary is available here and is compiled from the source you see below. Have fun and tell me if you find bugs or if it’s useful.


using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Security.AccessControl;
using System.Diagnostics;
using System.Security.Principal;

namespace AlexDuggleby.Tools.MirrorNTFSPermissions
{
    class Program
    {
        #region Properties

        /// <summary>
        /// The root we are copying from
        /// </summary>
        private static DirectoryInfo m_diFromRoot;

        /// <summary>
        /// The root we are copying to
        /// </summary>
        private static DirectoryInfo m_diToRoot;

        /// <summary>
        /// The log file to write to
        /// </summary>
        private static FileInfo m_fiLog;

        #endregion

        #region Main Operation

        static void Main(string[] args)
        {
            // Check args and output usage if required
            if (args.Length != 3)
            {
                OutputUsage();
                System.Environment.Exit(-1);
                return;
            }

            if (PopulateAndCheckDirectoriesExist(args) && ConfirmUserAction())
            {
                try
                {
                    Trace.Listeners.Add(new ConsoleTraceListener());
                    Trace.Listeners.Add(new TextWriterTraceListener(m_fiLog.FullName));

                    WriteTraceHeader();

                    Console.Clear();
                    CopyPermissions(m_diFromRoot, m_diToRoot, true);
                    WriteTraceFooter();
                }
                catch (ApplicationException aex)
                {
                    OutputError("Application", aex);
                }
                catch (Exception ex)
                {
                    OutputError("Critical", ex);
                }
            }

        }

        private static void CopyDirectorySecurity(DirectoryInfo diFrom, DirectoryInfo diTo, bool skipRootNameCheck, out bool continueInThisDirectory)
        {
            continueInThisDirectory = true;

            try
            {
                // Name comparison
                if (diFrom.Name != diTo.Name && !skipRootNameCheck)
                    throw new ApplicationException(string.Format("During processing the directory names changed! '{0}'-'{1}'",
                        diFrom.Name, diTo.Name));

                // Set my own permissions
                DirectorySecurity _dsFrom = Directory.GetAccessControl(diFrom.FullName, AccessControlSections.All);
                DirectorySecurity _dsTo = new DirectorySecurity();
                _dsTo.SetSecurityDescriptorSddlForm(_dsFrom.GetSecurityDescriptorSddlForm(AccessControlSections.All));
                Directory.SetAccessControl(diTo.FullName, _dsTo);

                // Tell the user what we are doing
                Trace.WriteLine(string.Format("Copying directory security from {0} to {1}",
                        diFrom.FullName, diTo.FullName));
            }
            catch (PathTooLongException _plex)
            {
                Trace.WriteLine(string.Format("[WARN] Path too long do continue with this directory '{0}'",
                        diFrom.FullName));
                continueInThisDirectory = false; // Abort because all files and subdirs will be affected
            }
            catch (Exception _ex) // General catch here, because we would like the batch to continue
            {
                Trace.WriteLine(string.Format("[ERROR] Error while doing current directory '{0}': {1}",
                        diFrom.FullName, _ex.ToString()));
                // We will continue here, since error type is undefined.
                continueInThisDirectory = true;
            }
        }

        #endregion

        #region File/Directory I/O and Security

        private static void CopyPermissions(DirectoryInfo diFrom, DirectoryInfo diTo, bool skipRootNameCheck)
        {
            bool _continueInThisDirectory;

            CopyDirectorySecurity(diFrom, diTo, skipRootNameCheck, out _continueInThisDirectory);

            if (_continueInThisDirectory)
            {
                // Now set files
                foreach (FileInfo _fiTo in diTo.GetFiles())
                {
                    CopyFileSecurity(diFrom, _fiTo);
                }

                // Get all the -to- subdirs ...
                foreach (DirectoryInfo _toSubDir in diTo.GetDirectories())
                {
                    // ... and check if they also exist in the from directory
                    DirectoryInfo _fromSubDir = diFrom.GetDirectories(_toSubDir.Name, SearchOption.TopDirectoryOnly).FirstOrDefault(d => d.Name == _toSubDir.Name);
                    if (_fromSubDir != null)
                    {
                        // Recursive
                        CopyPermissions(_fromSubDir, _toSubDir, false);
                    }
                }
            }
        }

        private static void CopyFileSecurity(DirectoryInfo diFrom, FileInfo fiTo)
        {
            // Turn the paths into relative paths based on the root dirs we started from
            string _relativeFromPath = fiTo.Directory.FullName.ToLower()
                .Replace(m_diToRoot.FullName.ToLower(), "");
            string _relativeToPath = diFrom.FullName.ToLower()
                .Replace(m_diFromRoot.FullName.ToLower(), "");

            // now compare those relative paths, just to be sure
            if (_relativeFromPath != _relativeToPath)
                throw new ApplicationException(string.Format(
                    "Subdirectories went out of sync during processing! '{0}'-'{1}'", _relativeFromPath, _relativeToPath));

            FileInfo _fiFrom = new FileInfo(Path.Combine(diFrom.FullName, fiTo.Name));

            if (_fiFrom.Exists) // if it doesn't exist in the from we will ignore...
            {
                // Copy the file permissions
                FileSecurity _fsFrom = File.GetAccessControl(_fiFrom.FullName);

                // Sometimes there are now access rules on the item, so only the owner can access it
                if (File.GetAccessControl(fiTo.FullName).GetAccessRules(true, true, typeof(NTAccount)).Count == 0)
                {
                    // Tell the user what we are doing
                    Trace.WriteLine(string.Format("File can only be accessed by owner: {0}", fiTo.FullName));
                }
                else
                {
                    try
                    {
                        // Copy the file permissions
                        FileSecurity _fsTo = new FileSecurity();
                        _fsTo.SetSecurityDescriptorSddlForm(_fsFrom.GetSecurityDescriptorSddlForm(AccessControlSections.All));
                        File.SetAccessControl(fiTo.FullName, _fsTo);

                        // Tell the user what we are doing
                        Trace.WriteLine(string.Format("Copying file security from {0} to {1}",
                            _fiFrom.FullName, fiTo.FullName));
                    }
                    catch (Exception _ex)
                    {
                        // Tell the user what went wrong
                        Trace.WriteLine(string.Format("[ERROR] Error while copying file security from {0} to {1}: {2}",
                            _fiFrom.FullName, fiTo.FullName, _ex.ToString()));
                    }
                }
            }
            else
            {
                Trace.WriteLine(string.Format("File did not exist in original tree: {0}",
                 fiTo.FullName));
            }
        }

        #endregion

        #region User I/O and Tracing

        private static void WriteTraceHeader()
        {
            Trace.WriteLine("*****************************************");
            Trace.WriteLine("Mirror NTFS Permissions");
            Trace.WriteLine("Version 1.0 - 27.05.2008");
            Trace.WriteLine("Alex Duggleby - http://alexduggleby.com");
            Trace.WriteLine("*****************************************");
            Trace.WriteLine("== New run @ " + DateTime.Now.ToString() + "");
            Trace.WriteLine("== From : " + m_diFromRoot.FullName + "");
            Trace.WriteLine("== To   : " + m_diToRoot.FullName + "");
            Trace.WriteLine("=========================================");
        }

        private static void WriteTraceFooter()
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Trace.WriteLine("=========================================");
            Trace.WriteLine("Finished!");
            Trace.WriteLine("=========================================");
            Console.ForegroundColor = ConsoleColor.White;
            Trace.Flush();
        }

        private static void OutputError(string errType, Exception ex)
        {
            Console.ForegroundColor = ConsoleColor.Red;
            string _s = string.Format("***********************{1}{2} error: {0}", ex.ToString(), Environment.NewLine, errType);
            Trace.WriteLine(_s);
            Console.ForegroundColor = ConsoleColor.White;
            OutputPressKeyToExit();
            Trace.Flush();
        }

        private static void OutputPressKeyToExit()
        {
            Console.WriteLine();
            Console.ForegroundColor = ConsoleColor.White;
            Console.WriteLine("{0}Press any key to exit", Environment.NewLine);
            Console.ReadKey();
        }

        private static void OutputUsage()
        {
            Console.WriteLine("Mirror NTFS Permissions");
            Console.WriteLine("by Alex Duggleby - http://alexduggleby.com");
            Console.WriteLine();
            Console.WriteLine("Usage: application.exe <path1> <path2> <log-file>");
            Console.WriteLine("Replaces all permissions on all subdirectory and files in path2 with those from path1 if they exist there.");
            Console.WriteLine("The <log-file> is used to write the appplications trace output.");
            Console.WriteLine();
        }

        private static bool ConfirmUserAction()
        {
            Console.WriteLine("Do you wish to copy all permissions form '{0}' to '{1}'? [y/n]",
                m_diFromRoot.FullName,
                m_diToRoot.FullName);

            if (Console.ReadKey().KeyChar.ToString().ToLower() != "y")
            {
                Console.WriteLine("User aborted!");
                OutputPressKeyToExit();
                System.Environment.Exit(-1);
                return false;
            }

            return true;
        }

        #endregion

        #region Prechecks

        private static bool PopulateAndCheckDirectoriesExist(string[] args)
        {
            string _dirFrom = args[0];
            string _dirTo = args[1];
            string _logFile = args[2];

            if (_dirFrom.EndsWith("\\")) _dirFrom = _dirFrom.Substring(0, _dirFrom.Length - 1);
            if (_dirTo.EndsWith("\\")) _dirTo = _dirTo.Substring(0, _dirTo.Length - 1);

            m_diFromRoot = new DirectoryInfo(_dirFrom);
            m_diToRoot = new DirectoryInfo(_dirTo);
            m_fiLog = new FileInfo(_logFile);

            if (!m_fiLog.Exists)
            {
                try
                {
                    m_fiLog.Create().Close();
                }
                catch (Exception _ex)
                {
                    Console.WriteLine("Could not create log file: " + _ex.ToString());
                }

                m_fiLog.Refresh();
            }

            if (!m_diFromRoot.Exists)
                Console.WriteLine("Directory '{0}' does not exist", m_diFromRoot.FullName);

            if (!m_diToRoot.Exists)
                Console.WriteLine("Directory '{0}' does not exist", m_diToRoot.FullName);

            if (!m_fiLog.Exists)
                Console.WriteLine("File '{0}' does not exist", m_fiLog.FullName);

            return m_diFromRoot.Exists && m_diToRoot.Exists && m_fiLog.Exists;
        }

        #endregion
    }
}

SkyDrive: Why aren’t thou nicer to me? or “How to crawl pages and download all images using C#”

Ok, SkyDrive is still a baby, and personally I’ve used other services providing file space in the cloud and enjoyed them a little better. But this post isn’t about if SkyDrive is good or bad, it’s just about a missing feature that is very painful. Someone wanted to share some fotos, uploaded them to SkyDrive and all I wanted was to download them all to my PC. Tough look, you can click on each and every image to get to the preview page, where you click on the preview picture to then finally get at the actual picture. Multiply that by about 100. I have better things to do than waste my time on that.

So a Dev does what he does best, fires up Visual Studio 2008 and hacks away (did I just say I had something better to do - well I lied partially, but before I go off to do that, there is always time for some good ol’ C#).

I’ve posted it here not as a finished utility (there are no binaries) but as a small sample. Using WebClients, RegEx and some other stuff it downloads the list page of the SkyDrive folder, fetches the preview page and then downloads the actual image to a folder on the hard disk. Not really rocket science and of course there are a few quirks (no real error handling for example), but it’s just a sample. Feel free to extend as you wish, don’t blame me if it starts downloading Gigabytes of files overnight, because you accidentally crawled a HoneyPot. (And yes, it only downloads jpgs at the moment. I didn’t need any other types.)

May those SkyDrive bytes be with you…


/**********************************************************************************
 *
 * Example Application for crawling web pages and downloading images.
 *
 * This code works if you pass in a SkyDrive Folder Url (http://.... /browse.aspx/...)
 * and will download any jpg images it finds in there.
 *
 * Permission to use, copy, modify, distribute and sell this software and its
 * documentation for any purpose is hereby granted without fee.
 * I make no representations about the suitability of this software for any purpose.
 * It is provided "as is" without express or implied warranty.
 *
 * Alex Duggleby - 24.05.08 - V0.9 - http://alexduggleby.com
 *
 **********************************************************************************/
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.Text.RegularExpressions;
using System.IO;
using System.Web;
using System.ComponentModel;

namespace Tools.SkyDrive.DownloadAll
{
    class Program
    {
        // Used for tracking how many items we have left
        private static int _wcInnerCount = 0;
        private static int _wcInnerCompleted = 0;

        // We have to start somewhere
        private static Uri _uriStart;

        // Work we have already done
        private readonly static List<string> _urisCrawled = new List<string>();
        private readonly static List<string> _imagesDownloaded = new List<string>();

        // Download images to?
        private readonly static DirectoryInfo _diDownloadTo = new DirectoryInfo(Path.Combine(Path.Combine(System.Environment.GetFolderPath(Environment.SpecialFolder.Personal), "Downloads"),"Images"));

        // This finds urls in the page
        private readonly static Regex _regexUrl = new Regex("href\\s*=\\s*(?:(?:\\\"(?<url>[^\\\"]*)\\\")|(?<url>[^\\s]* ))");

        // This finds the open url in the image page
        private readonly static Regex _regexUrlOpen = new Regex("href\\s*=\\s*(?:(?:\\\"(?<url>[^\\\"]*)\\\")|(?<url>[^\\s]*)) title=\\\"Open\\\""); 

        /// <summary>
        /// Takes the url to a skydrive folder page and downloads all jpg images.
        /// </summary>
        static void Main(string[] args)
        {
            // Usage check
            if (args.Length != 1)
            {
                Console.WriteLine("Usage: App.exe http://theUrlToThe/SkyDrive/FolderPage");
                return;
            }

            try
            {
                // First parameter is url
                _uriStart = new Uri(args[0]);
            }
            catch (Exception _ex)
            {
                Console.WriteLine("Invalid Url. " + _ex.Message);
                return;
            }

            // Make sure download directory exists
            if (!_diDownloadTo.Exists) _diDownloadTo.Create();

            using (WebClient _wc = new WebClient())
            {
                // This is the index with all the images
                string _pageContents = _wc.DownloadString(_uriStart);

                // Each image has a preview page, so we get the url to that, before we get the url to the actual image
                foreach (Match _matchUrlToImagePage
                    in _regexUrl.Matches(_pageContents))
                {
                    Uri _uriToImagePage =
                        new Uri(_uriStart, HttpUtility.HtmlDecode(_matchUrlToImagePage.Groups["url"].Value));

                    CrawlPreviewPage(_uriToImagePage);
                }
            }

            // Wait for the async web clients to complete...
            while (_wcInnerCompleted < _wcInnerCount)
            {
                Console.WriteLine("Wait for images to complete...");
                Console.ReadLine();
            }

            Console.WriteLine("Should be finished!");
            Console.ReadLine();
        }

        /// <summary>
        /// Parses the preview page and finds the actual image link
        /// </summary>
        /// <param name="uriToImagePage">The url to the preview page</param>
        /// <returns></returns>
        private static void CrawlPreviewPage(Uri uriToImagePage)
        {
            using (WebClient _wc = new WebClient())
            {
                if (!_urisCrawled.Contains(uriToImagePage.ToString()))
                {
                    _urisCrawled.Add(uriToImagePage.ToString());

                    if (uriToImagePage.ToString().ToLower().EndsWith(".jpg"))
                    {
                        string _pageContents = _wc.DownloadString(uriToImagePage);

                        // Find the image we want to download... There should be
                        // only one link with title="Open" in it.
                        foreach (Match _matchImage in _regexUrlOpen.Matches(_pageContents))
                        {
                            Uri _uriToImage = new Uri(_matchImage.Groups["url"].Value);

                            DownloadImage(_uriToImage);
                        }
                    }
                }
            }
        }

        /// <summary>
        /// Downloads async'ly an image from a Uri
        /// </summary>
        /// <param name="uriToImage">The uri to download</param>
        private static void DownloadImage(Uri uriToImage)
        {
            // Output the url
            Console.WriteLine("{0}{1}", uriToImage.ToString(), Environment.NewLine);

            if (!_imagesDownloaded.Contains(uriToImage.ToString()))
            {
                _imagesDownloaded.Add(uriToImage.ToString());
                string _lowerUrl = uriToImage.ToString().ToLower();

                // Simple checking
                if (_lowerUrl.EndsWith(".jpg") &&
                   (!_lowerUrl.Contains("browse")) &&
                   (!_lowerUrl.Contains("self")))
                {
                    // HtmlDecode here because some urls have encoded characters
                    string _localFilename = HttpUtility.HtmlDecode(
                        uriToImage.Segments[uriToImage.Segments.Length - 1]);

                    // Create a valid local filename
                    Path.GetInvalidPathChars().ToList().ForEach(
                        c => _localFilename = _localFilename.Replace(c, '_'));

                    Console.Write("Downloading {0}...{1}", _localFilename, Environment.NewLine);

                    // Create a seperate web client for each image (uses async, and you can't
                    // issue two downloads at the same time for the same client). Of course
                    // here we should be using some kind of pooling but this is the quickest
                    // way to do it.
                    using (WebClient _wcInner = new WebClient())
                    {
                        _wcInnerCount++;
                        _wcInner.DownloadFileAsync(uriToImage, Path.Combine(_diDownloadTo.ToString(), _localFilename));
                        _wcInner.DownloadFileCompleted += new AsyncCompletedEventHandler(_wcInner_DownloadFileCompleted);
                    }
                }
            }
        }

        // Is fired when a download complete. We output status and check if we are finished!
        private static void _wcInner_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e)
        {
            // Increase the completed counter
            _wcInnerCompleted++;

            // Ok, we could do some more extensive checking, this could trigger
            // even if there are still items to download... but hey, it's just a
            // quick utility!
            if (_wcInnerCompleted == _wcInnerCount)
            {
                Console.WriteLine("{0}{1}{2}", Environment.NewLine, "Finished all files!", Environment.NewLine);
                Console.ReadLine();
            }
            else
            {
                Console.WriteLine("File {0} of {1} completed!", _wcInnerCompleted, _wcInnerCount);
            }
        }
    }
}

Off-Topic: T-SQL: Replace all occurrences in all columns in all tables?

Update: You can download the sql file here: http://cid-8e9963932ac1f048.skydrive.live.com/self.aspx/%C3%96ffentlich/STRING|_REPLACER|_V1.2.sql (saves you cleaning up the copy-pasted code from below… oh and hello experts-exchange-users)

Have you ever been asked to write something where you knew from the start that it could possibly wreak havok? Well I did today and since I haven’t posted in a while I thought I’d quickly share. Don’t worry I’ll be back blogging Sync, MVC and more soon (and hopefully tell you why I was gone).

Without talking too much about the real scenario that this was needed for, let me just say there is a large database that stores configuration data including a set of connection strings in multiple places spread across numerous tables. And our admins moved the database but didn’t know where exactly to change these config strings. I was the only dev left in the office at the time and they turned to me asking me if I could write a script that “replaced all occurrences of stringA with stringB throughout the whole database”.

Yikes. It sounds like trouble, you know it’s gonna hurt and I don’t want to be anywhere nearby when they press F5 to run the script. Ok, nevertheless I put together a little script that did just that, sent it to them and let’s see what happens next week. They have backups so even if everything goes wrong we can quickly restore things.

Here’s what I came up with. It’s not rocket science, but does the job. Use the script at your own risk (I didn’t find anything comparable on the web so maybe it will come in handy to someone). It shows some interesting usage of the system tables that I use now and again for things like this (when I don’t have my beloved SQLSMO). Have fun and use with caution (I can’t say that enough!)


------------------------------------------------------------
-- Name: STRING REPLACER
-- Author: ADUGGLEBY
-- Version: 20.05.2008 (1.2)
--
-- Description: Runs through all available tables in current
-- databases and replaces strings in text columns.
------------------------------------------------------------

-- PREPARE
SET NOCOUNT ON

-- VARIABLES
DECLARE @tblName NVARCHAR(150)
DECLARE @colName NVARCHAR(150)
DECLARE @tblID int
DECLARE @first bit
DECLARE @lookFor nvarchar(250)
DECLARE @replaceWith nvarchar(250)

-- CHANGE PARAMETERS
SET @lookFor = 'virtual2'
SET @replaceWith = 'virtual3'

-- TEXT VALUE DATA TYPES
DECLARE @supportedTypes TABLE ( xtype NVARCHAR(20) )
INSERT INTO @supportedTypes SELECT XTYPE FROM SYSTYPES WHERE NAME IN ('varchar','char','nvarchar','nchar','xml')

-- ALL USER TABLES
DECLARE cur_tables CURSOR FOR
SELECT SO.name, SO.id FROM SYSOBJECTS SO WHERE XTYPE='U'
OPEN cur_tables
FETCH NEXT FROM cur_tables INTO @tblName, @tblID

WHILE @@FETCH_STATUS = 0
BEGIN
	-------------------------------------------------------------------------------------------
	-- START INNER LOOP - All text columns, generate statement
	-------------------------------------------------------------------------------------------
	DECLARE @temp NVARCHAR(4000)
	DECLARE @count INT
	SELECT @count = COUNT(name) FROM SYSCOLUMNS WHERE ID = @tblID AND
		XTYPE IN (SELECT xtype FROM @supportedTypes)

	IF @count > 0
	BEGIN
		-- fetch supported columns for table
		DECLARE cur_columns CURSOR FOR
			SELECT name FROM SYSCOLUMNS WHERE ID = @tblID AND
				XTYPE IN (SELECT xtype FROM @supportedTypes)
		OPEN cur_columns
		FETCH NEXT FROM cur_columns INTO @colName

		-- generate opening UPDATE cmd
		SET @temp = '
	PRINT ''Replacing ' + @tblName + '''

	UPDATE ' + @tblName + ' SET
		'
		SET @first = 1

		-- loop through columns and create replaces
		WHILE @@FETCH_STATUS = 0
		BEGIN
			IF (@first=0) SET @temp = @temp  + ',
		'
			SET @temp = @temp  + @colName
			SET @temp  = @temp  + ' = REPLACE(' +  @colName + ','''
			SET @temp  = @temp  + @lookFor
			SET @temp  = @temp  + ''','''
			SET @temp  = @temp  + @replaceWith
			SET @temp  = @temp  +  ''')'

			SET @first = 0
			FETCH NEXT FROM cur_columns INTO @colName
		END

		PRINT @temp

		CLOSE cur_columns
		DEALLOCATE cur_columns
	END
	-------------------------------------------------------------------------------------------
	-- END INNER
	-------------------------------------------------------------------------------------------

	FETCH NEXT FROM cur_tables INTO @tblName, @tblID
END

CLOSE cur_tables
DEALLOCATE cur_tables


Subscribe / Search

Imagine Cup 2009 - Egypt

msplogo_small.jpg

mcprgb.png

XING

a

 

May 2008
M T W T F S S
« Apr   Jun »
 1234
567891011
12131415161718
19202122232425
262728293031  

Blog Stats

  • 26,740 hits