@article{bamford-ijrmms21,
title = {A deep learning approach for rock fragmentation analysis},
author = {Thomas Bamford and Kamran Esmaeili and Angela P. Schoellig},
journal = {{International Journal of Rock Mechanics and Mining Sciences}},
year = {2021},
volume = {145},
doi = {10.1016/j.ijrmms.2021.104839},
pages = {104839},
urllink = {https://www.sciencedirect.com/science/article/pii/S1365160921002239},
abstract = {In mining operations, blast-induced rock fragmentation affects the productivity and efficiency of downstream operations including digging, hauling, crushing, and grinding. Continuous measurement of rock fragmentation is essential for optimizing blast design. Current methods of rock fragmentation analysis rely on either physical screening of blasted rock material or image analysis of the blasted muckpiles; both are time consuming. This study aims to present and evaluate the measurement of rock fragmentation using deep learning strategies. A deep neural network (DNN) architecture was used to predict characteristic sizes of rock fragments from a 2D image of a muckpile. The data set used for training the DNN model is composed of 61,853 labelled images of blasted rock fragments. An exclusive data set of 1,263 labelled images were used to test the DNN model. The percent error for coarse characteristic size prediction ranges within ±25% when evaluated using the test set. Model validation on orthomosaics for two muckpiles shows that the deep learning method achieves a good accuracy (lower mean percent error) compared to manual image labelling. Validation on screened piles shows that the DNN model prediction is similar to manual labelling accuracy when compared with sieving analysis.},
}

@article{medinac-mme20,
title = {Haul road monitoring in open pit mines using unmanned aerial vehicles: A case study at {Bald Mountain} mine site},
author = {Filip Medinac and Thomas Bamford and Matthew Hart and Michal Kowalczyk and Kamran Esmaeili},
journal = {{Mining, Metallurgy \& Exploration}},
year = {2020},
volume = {37},
pages = {1877--1883},
doi = {10.1007/s42461-020-00291-w},
urllink = {https://rdcu.be/cbOuJ},
abstract = {Improved haul road conditions can positively impact mine operations resulting in increased safety, productivity gains, increased tire life, and lower maintenance costs. For these reasons, a monitoring program is required to ensure the operational efficiency of the haul roads. Currently, at Bald Mountain mine, monthly site severity studies, ad hoc inspections by frontline supervisors, or operator feedback reporting is used to assess road conditions. These methods are subjective and provide low temporal resolution data. This case study presents novel unmanned aerial vehicle (UAV) technologies, applied on a critical section of haul road at Bald Mountain, to showcase the potential for monitoring haul roads. The results show that orthophotos and digital elevation models can be used to assess the road smoothness condition and to check the road design compliance. Moreover, the aerial mapping allows detection of surface water, rock spillage, and potholes on the road that can be quickly repaired/removed by the dedicated road maintenance team.},
}

@article{bamford-rs20,
title = {Continuous Monitoring and Improvement of the Blasting Process in Open Pit Mines Using Unmanned Aerial Vehicle Techniques},
author = {Thomas Bamford and Filip Medinac and Kamran Esmaeili},
journal = {{Remote Sensing}},
year = {2020},
volume = {12},
number = {17},
doi = {10.3390/rs12172801},
pages = {2801},
urllink = {https://www.mdpi.com/2072-4292/12/17/2801},
abstract = {The current techniques used for monitoring the blasting process in open pit mines are manual, intermittent and inefficient and can expose technical manpower to hazardous conditions. This study presents the application of unmanned aerial vehicle (UAV) systems for monitoring and improving the blasting process in open pit mines. Field experiments were conducted in different open pit mines to assess rock fragmentation, blast-induced damage on final pit walls, blast dynamics and the accuracy of blastholes including production and pre-split holes. The UAV-based monitoring was done in three different stages, including pre-blasting, blasting and post-blasting. In the pre-blasting stage, pit walls were mapped to collect structural data to predict in situ block size distribution and to develop as-built pit wall digital elevation models (DEM) to assess blast-induced damage. This was followed by mapping the production blasthole patterns implemented in the mine to investigate drillhole alignment. To monitor the blasting process, a high-speed camera was mounted on the UAV to investigate blast initiation, sequencing, misfired holes and stemming ejection. In the post-blast stage, the blasted rock pile (muck pile) was monitored to estimate fragmentation and assess muck pile configuration, heave and throw. The collected aerial data provide detailed information and high spatial and temporal resolution on the quality of the blasting process and significant opportunities for process improvement. The current challenges with regards to the application of UAVs for blasting process monitoring are discussed, and recommendations for obtaining the most value out of an UAV application are provided.},
}

@INPROCEEDINGS{medinac-dfne18,
author = {Filip Medinac and Thomas Bamford and Kamran Esmaeili and Angela P. Schoellig},
title = {Pre- and post-blast rock block size analysis using {UAV-Lidar} based data and discrete fracture network},
booktitle = {{Proc. of the 2nd International Discrete Fracture Network Engineering (DFNE)}},
year = {2018},
abstact = {Drilling and blasting is one of the key processes in open pit mining, required to reduce in-situ rock block size to rock fragments that can be handled by mine equipment. It is a significant cost driver of any mining operation which can influence the downstream mining processes. In-situ rock block size influences the muck pile size distribution after blast, and the amount of drilling and explosive required to achieve a desired distribution. Thus, continuous measurement of pre- and post-blast rock block size distribution is essential for the optimization of the rock fragmentation process. This paper presents the results of a case study in an open pit mine where an Unmanned Aerial Vehicle (UAV) was used for mapping of the pit walls before blast. Pit wall mapping and aerial data was used as input to generate a 3D Discrete Fracture Network (DFN) model of the rock mass and to estimate the in-situ block size distribution. Data collected by the UAV was also used to estimate the post-blast rock fragment size distribution. The knowledge of in-situ and blasted rock size distributions can be related to assess blast performance. This knowledge will provide feedback to production engineers to adjust the blast design.},
}

@inproceedings{bamford-fragblast12,
author = {Thomas Bamford and Kamran Esmaeili and Angela P. Schoellig},
title = {Evaluation of {UAV} system accuracy for automated fragmentation measurement},
booktitle = {{Proc. of the 12th International Symposium on Rock Fragmentation by Blasting (FRAGBLAST)}},
year = {2018},
pages = {715--730},
abstract = {The current practice of collecting rock fragmentation data is highly manual and provides data with low temporal and spatial resolution. Unmanned Aerial Vehicle (UAV) technology can increase both temporal and spatial data resolution without exposing technicians to hazardous conditions. Our previous works using UAV technology to acquire real-time rock fragmentation data has shown comparable quality results to sieving in a lab environment. However, when applied to a mining environment, it is essential to quantify the accuracy of scale estimation and rock size distribution by considering various sources of uncertainties such as the UAV GPS, which provides noisy measurements. In the current paper, we investigate the accuracy of application of UAVs to collect photographic data for fragmentation analysis. This is done by evaluating the accuracy of the 3D model generated using the UAV system, estimated image scale, and the measured rock size distribution. This paper also investigates the impact of flight altitude on the measured rock size distribution.},
}

@article{bamford-ijmre17,
title = {A Real-Time Analysis of Post-Blast Rock Fragmentation Using {UAV} Technology},
author = {Bamford, Thomas and Esmaeili, Kamran and Schoellig, Angela P.},
journal = {{International Journal of Mining, Reclamation and Environment}},
year = {2017},
volume = {31},
number = {6},
doi = {10.1080/17480930.2017.1339170},
pages = {439--456},
publisher = {Taylor & Francis},
urlvideo = {https://youtu.be/q0syk6J_JHY},
abstract = {The current practice of collecting rock fragmentation data for image analysis is highly manual and provides data with low temporal and spatial resolution. Using Unmanned Aerial Vehicles (UAVs) for collecting images of rock fragments improves the quality of the image data and automates the data collection process. This work presents the results of laboratory-scale using a UAV. The goal is to highlight the benefits of aerial fragmentation analysis in terms of both prediction accuracy and time effort. The pile was manually photographed and the results of the manual method were compared to the UAV method.},
}

@INPROCEEDINGS{bamford-apcom17,
author={Thomas Bamford and Kamran Esmaeili and Angela P. Schoellig},
title={Aerial Rock Fragmentation Analysis in Low-Light Condition Using {UAV} Technology},
booktitle={{Proc. of Application of Computers and Operations Research in the Mining Industry (APCOM)}},
year={2017},
pages = {4-1--4-8},
urlslides={../../wp-content/papercite-data/slides/bamford-apcom17-slides.pdf},
abstract={In recent years, Unmanned Aerial Vehicle (UAV) technology has been introduced into the mining industry to conduct terrain surveying. This work investigates the application of UAVs with artificial lighting for measurement of rock fragmentation under poor lighting conditions, representing night shifts in surface mines or working conditions in underground mines. The study relies on indoor and outdoor experiments for rock fragmentation analysis using a quadrotor UAV. Comparison of the rock size distributions in both cases show that adequate artificial lighting enables similar accuracy to ideal lighting conditions.},
}

@TECHREPORT{bamford-iros17,
title = {Point-cloud-based aerial fragmentation analysis for application in the minerals industry},
institution = {arXiv},
author = {Thomas Bamford and Kamran Esmaeili and Angela P. Schoellig},
year = {2017},
urllink = {https://arxiv.org/abs/1703.01945},
abstract = {This work investigates the application of Unmanned Aerial Vehicle (UAV) technology for measurement of rock fragmentation without placement of scale objects in the scene to determine image scale. Commonly practiced image-based rock fragmentation analysis requires a technician to walk to a rock pile, place a scale object of known size in the area of interest, and capture individual 2D images. Our previous work has used UAV technology for the first time to acquire real-time rock fragmentation data and has shown comparable quality of results; however, it still required the (potentially dangerous) placement of scale objects, and continued to make the assumption that the rock pile surface is planar and that the scale objects lie on the surface plane. This work improves our UAV-based approach to enable rock fragmentation measurement without placement of scale objects and without the assumption of planarity. This is achieved by first generating a point cloud of the rock pile from 2D images, taking into account intrinsic and extrinsic camera parameters, and then taking 2D images for fragmentation analysis. This work represents an important step towards automating post-blast rock fragmentation analysis. In experiments, a rock pile with known size distribution was photographed by the UAV with and without using scale objects. For fragmentation analysis without scale objects, a point cloud of the rock pile was generated and used to compute image scale. Comparison of the rock size distributions show that this point-cloud-based method enables producing measurements with better or comparable accuracy (within 10% of the ground truth) to the manual method with scale objects.},
}

@INPROCEEDINGS{bamford-cami16,
author = {Thomas Bamford and Kamran Esmaeili and Angela P. Schoellig},
title = {A real-time analysis of rock fragmentation using {UAV} technology},
booktitle = {{Proc. of the International Conference on Computer Applications in the Minerals Industries (CAMI)}},
year = {2016},
urllink = {http://arxiv.org/abs/1607.04243},
urlvideo = {https://youtu.be/q0syk6J_JHY},
urlslides={../../wp-content/papercite-data/slides/bamford-cami16-slides.pdf},
abstract = {Accurate measurement of blast-induced rock fragmentation is of great importance for many mining operations. The post-blast rock size distribution can significantly influence the efficiency of all the downstream mining and comminution processes. Image analysis methods are one of the most common methods used to measure rock fragment size distribution in mines regardless of criticism for lack of accuracy to measure fine particles and other perceived deficiencies. The current practice of collecting rock fragmentation data for image analysis is highly manual and provides data with low temporal and spatial resolution. Using Unmanned Aerial Vehicles (UAVs) for collecting images of rock fragments can not only improve the quality of the image data but also automate the data collection process. Ultimately, real-time acquisition of high temporal- and spatial-resolution data based on UAV technology will provide a broad range of opportunities for both improving blast design without interrupting the production process and reducing the cost of the human operator.},
}

@MISC{bamford-ops16,
author = {Thomas Bamford and Kamran Esmaeili and Angela P. Schoellig},
title = {Rock fragmentation analysis using {UAV} technology},
year = {2016},
volume = {59},
number = {4},
pages = {14-16},
howpublished = {Professional Magazine Article, Ontario Professional Surveyor (OPS) Magazine, Assn. of Ontario Land Surveyors},
urllink = {http://publications.aols.org/OPS-Magazine/2016Fall/},
}

@article{kayani-jcce23,
author = {Navid Kayhani and Angela Schoellig and Brenda McCabe},
title = {Perception-Aware Tag Placement Planning for Robust Localization of {UAVs} in Indoor Construction Environments},
journal = {{Journal of Computing in Civil Engineering}},
volume = {37},
number = {2},
pages = {04022060},
year = {2023},
doi = {10.1061/JCCEE5.CPENG-5068},
urllink = {https://ascelibrary.org/doi/abs/10.1061/JCCEE5.CPENG-5068},
abstract = {Tag-based visual-inertial localization is a lightweight method for enabling autonomous data collection missions of low-cost unmanned aerial vehicles (UAVs) in indoor construction environments. However, finding the optimal tag configuration (i.e., number, size, and location) on dynamic construction sites remains challenging. This work proposes a perception-aware genetic algorithm-based tag placement planner (PGA-TaPP) to determine the optimal tag configuration using four-dimensional (4D) building information models (BIM), considering the project progress, safety requirements, and UAV’s localizability. The proposed method provides a 4D plan for tag placement by maximizing the localizability in user-specified regions of interest (ROIs) while limiting the installation costs. Localizability is quantified using the Fisher information matrix (FIM) and encapsulated in navigable grids. The experimental results show the effectiveness of our method in finding an optimal 4D tag placement plan for the robust localization of UAVs on under-construction indoor sites.}
}

@article{kayhani-autocon22,
author = {Navid Kayhani and Wenda Zhao and Brenda McCabe and Angela P. Schoellig},
title = {Tag-based visual-inertial localization of unmanned aerial vehicles in indoor construction environments using an on-manifold extended {Kalman} filter},
journal = {{Automation in Construction}},
year = {2022},
volume = {135},
pages = {104112},
issn = {0926-5805},
doi = {https://doi.org/10.1016/j.autcon.2021.104112},
url = {https://www.sciencedirect.com/science/article/pii/S092658052100563X},
keywords = {Indoor localization, Unmanned aerial vehicle, Extended Kalman filter, SE(3), On-manifold state estimation, Autonomous navigation, Building information model, Construction robotics, AprilTag},
abstract = {Automated visual data collection using autonomous unmanned aerial vehicles (UAVs) can improve the accessibility and accuracy of the frequent data required for indoor construction inspections and tracking. However, robust localization, as a critical enabler for autonomy, is challenging in ever-changing, cluttered, GPS-denied indoor construction environments. Rapid alterations and repetitive low-texture areas on indoor construction sites jeopardize the reliability of typical vision-based solutions. This research proposes a tag-based visual-inertial localization method for off-the-shelf UAVs with only a camera and an inertial measurement unit (IMU). Given that tag locations are known in the BIM, the proposed method estimates the UAV's global pose by fusing inertial data and tag measurements using an on-manifold extended Kalman filter (EKF). The root-mean-square error (RMSE) achieved in our experiments in laboratory and simulation, being as low as 2 − 5 cm, indicates the potential of deploying the proposed method for autonomous navigation of low-cost UAVs in indoor construction environments.}
}

@INPROCEEDINGS{kayhani-csce22,
author={Navid Kayhani and Brenda McCabe and Angela P. Schoellig},
booktitle={{Proc. of the Canadian Society of Civil Engineering Annual Conference (CSCE)}},
title={Stochastic modeling of tag installation error for robust on-manifold tag-based visual-inertial localization},
year={2022},
note={Accepted},
urlvideo = {https://youtu.be/2frTKgOwbf4},
abstract = {Autonomous mobile robots, including unmanned aerial vehicles (UAVs), have received significant attention for their applications in construction. These platforms have great potential to automate and enhance the quality and frequency of the required data for many tasks such as construction schedule updating, inspections, and monitoring. Robust localization is a critical enabler for reliable deployments of autonomous robotic platforms. Automated robotic solutions rely mainly on the global positioning system (GPS) for outdoor localization. However, GPS signals are denied indoors, and pre-built environment maps are often used for indoor localization. This entails generating high-quality maps by teleoperating the mobile robot in the environment. Not only is this approach time-consuming and tedious, but it also is unreliable in indoor construction settings. Layout changes with construction progress, requiring frequent mapping sessions to support autonomous missions. Moreover, the effectiveness of vision-based solutions relying on visual features are highly impacted in low texture and repetitive areas on site. To address these challenges, we previously proposed a low-cost, lightweight tag-based visual-inertial localization method using AprilTags. Tags, in this method, are paper printable landmarks with known sizes and locations, representing the environment’s quasi-map. Since tag placement/replacement is a manual process, it is subject to human errors. In this work, we study the impact of human error in the manual tag installation process and propose a stochastic approach to account for this uncertainty using the Lie group theory. Employing Monte Carlo simulation, we experimentally show that the proposed stochastic model incorporated in our on-manifold formulation improves the robustness and accuracy of tag-based localization against inevitable imperfections in manual tag installation on site.}
}

@INPROCEEDINGS{kayhani-crc20,
author={Navid Kayhani and Brenda McCabe and Ahmed Abdelaal and Adam Heins and Angela P. Schoellig},
booktitle={{Proc. of the Construction Research Congress}},
title={Tag-based Indoor Localization of UAVs in Construction Environments: Opportunities and Challenges in Practice},
year={2020},
pages={226--235},
doi={10.1061/9780784482865.025},
urllink = {https://ascelibrary.org/doi/epdf/10.1061/9780784482865.025},
abstract = {Automated visual inspection and progress monitoring of construction projects using different robotic platforms have recently attracted scholars' attention. Unmanned/unoccupied aerial vehicles (UAVs), however, are more and more being used for this purpose because of their maneuverability and perspective capabilities. Although a multi-sensor autonomous UAV can enhance the collection of informative data in constantly-evolving construction environments, autonomous flight and navigation of UAVs are challenging in indoor environments where the global positioning system (GPS) might be denied or unreliable. In such continually changing environments, the limited external infrastructure and the existence of unknown obstacles are two key challenges that need to be addressed. On the other hand, construction indoor environments are not fully unknown, as a progressively updating building information model (BIM) provides valuable prior knowledge about the GPS-denied environment. This fact can potentially create unique opportunities to facilitate the indoor navigation process in construction projects. The authors have previously shown the potentials of AprilTag fiducial markers for localization of a camera-equipped UAV in various controlled experimental setups in the laboratory. In this paper, we investigate the opportunities and challenges of using tag-based localization techniques in real-world construction environments.}
}

@INPROCEEDINGS{kayhani-isarc19,
author = {Navid Kayhani and Adam Heins and Wenda Zhao and Mohammad Nahangi and Brenda McCabe and Angela P. Schoellig},
title = {Improved Tag-based Indoor Localization of {UAV}s Using Extended {Kalman} Filter},
booktitle = {{Proc. of the International Symposium on Automation and Robotics in Construction (ISARC)}},
year = {2019},
pages = {624--631},
doi={10.22260/ISARC2019/0083},
abstract = {Indoor localization and navigation of unmanned aerial vehicles (UAVs) is a critical function for autonomous flight and automated visual inspection of construction elements in continuously changing construction environments. The key challenge for indoor localization and navigation is that the global positioning system (GPS) signal is not sufficiently reliable for state estimation. Having used the AprilTag markers for indoor localization, we showed a proof-of-concept that a camera-equipped UAV can be localized in a GPS-denied environment; however, the accuracy of the localization was inadequate in some situations. This study presents the implementation and performance assessment of an Extended Kalman Filter (EKF) for improving the estimation process of a previously developed indoor localization framework using AprilTag markers. An experimental set up is used to assess the performance of the updated estimation process in comparison to the previous state estimation method and the ground truth data. Results show that the state estimation and indoor localization are improved substantially using the EKF. To have a more robust estimation, we extract and fuse data from multiple tags. The framework can now be tested in real-world environments given that our continuous localization is sufficiently robust and reliable.},
}

@INPROCEEDINGS{nahangi-isarc18,
author={Mohammad Nahangi and Adam Heins and Brenda McCabe and Angela P. Schoellig},
title={Automated Localization of {UAV}s in {GPS}-Denied Indoor Construction Environments Using Fiducial Markers},
booktitle = {{Proc. International Symposium on Automation and Robotics in Construction (ISARC)}},
year = {2018},
pages={88--94},
abstract = {Unmanned Aerial Vehicles (UAVs) have opened a wide range of opportunities and applications in different sectors including construction. Such applications include: 3D mapping from 2D images and video footage, automated site inspection, and performance monitoring. All of the above-mentioned applications perform well outdoors where GPS is quite reliable for localization and navigation of UAV’s. Indoor localization and consequently indoor navigation have remained relatively untapped, because GPS is not sufficiently reliable and accurate in indoor environments. This paper presents a method for localization of aerial vehicles in GPS-denied indoor construction environments. The proposed method employs AprilTags that are linked to previously known coordinates in the 3D building information model (BIM). Using cameras on-board the UAV and extracting the transformation from the tag to the camera’s frame, the UAV can be localized on the site. It can then use the previously computed information for navigation between critical locations on construction sites. We use an experimental setup to verify and validate the proposed method by comparing with an indoor localization system as the ground truth. Results show that the proposed method is sufficiently accurate to perform indoor navigation. Moreover, the method does not intensify the complexity of the construction execution as the tags are simply printed and placed on available surfaces at the construction site.},
}

@INPROCEEDINGS{foerster-iser18,
author={Julian F. M. Foerster and Mohamed K. Helwa and Xintong Du and Angela P. Schoellig},
title={Hybrid Model Predictive Control for Crosswind Stabilization of Hybrid Airships},
booktitle={{Proc. of the International Symposium on Experimental Robotics (ISER)}},
year={2018},
pages={499-510},
doi={10.1007/978-3-030-33950-0_43},
urllink={https://link.springer.com/chapter/10.1007/978-3-030-33950-0_43},
abstract={Hybrid airships are heavier-than-air vehicles that generate a majority of the lift using buoyancy. The resulting high energy efficiency during operation and short take-off and landing distances make this vehicle class very suited for a number of logistics applications. However, the range of safe operating conditions can be limited due to a high susceptibility to crosswinds during taxiing, take-off and landing. The goal of this work is to design and implement an automated counter-gust system (CGS) that stabilizes a hybrid airship against wind disturbances during ground operations by controlling thrusters that are mounted to the wingtips. The CGS controller should compute optimal control inputs, run autonomously without pilot intervention, be computationally efficient to run on onboard hardware, and be flexible regarding adaption to future aircraft.},
}

@TECHREPORT{wang-tr15,
author = {Xingbo Wang and Natasha Dalal and Tristan Laidlow and Angela P. Schoellig},
title = {A Flying Drum Machine},
year = {2015},
urlvideo={https://youtu.be/d5zG-BWB7lE?list=PLD6AAACCBFFE64AC5},
abstract = {This paper proposes the use of a quadrotor aerial vehicle as a musical instrument. Using the idea of interactions based on physical contact, a system is developed that enables humans to engage in artistic expression with a flying robot and produce music. A robotic user interface that uses physical interactions was created for a quadcopter. The interactive quadcopter was then programmed to drive playback of drum sounds in real-time in response to physical interaction. An intuitive mapping was developed between machine movement and art/creative composition. Challenges arose in meeting realtime latency requirements mainly due to delays in input detection. They were overcome through the development of a quick input detection method, which relies on accurate yet fast digital filtering. Successful experiments were conducted with a professional musician who used the interface to compose complex rhythm patterns. A video accompanying this paper demonstrates his performance.}
}